Rtop 78 create log filters

.gitignore

@@ -14,6 +14,7 @@ __pycache__/
 *.pem
 *.db
 *.socket
+*.installed
 
 # Ignore all object files and shared libraries
 *.o

webserver/app.py

@@ -28,16 +28,15 @@
     MAX_FILE_SIZE
 )
 
-# from logger import get_logger, LogParser
+from logger import get_logger, LogParser
 
+logger, _ = get_logger("logger", use_buffer=True)
 
 app = flask.Flask(__name__)
 app.secret_key = str(os.urandom(16))
 login_manager = flask_login.LoginManager()
 login_manager.init_app(app)
 
-# logger = get_logger(use_buffer=True)
-
 runtime_manager = RuntimeManager(
     runtime_path="./build/plc_main",
     plc_socket="/run/runtime/plc_runtime.socket",
@@ -63,7 +62,15 @@ def handle_stop_plc(data: dict) -> dict:
 
 
 def handle_runtime_logs(data: dict) -> dict:
-    response = runtime_manager.get_logs()
+    if "id" in data:
+        min_id = int(data["id"])
+    else:
+        min_id = None
+    if "level" in data:
+        level = data["level"]
+    else:
+        level = None
+    response = runtime_manager.get_logs(min_id=min_id, level=level)
     return {"runtime-logs": response}
 
 
@@ -200,12 +207,8 @@ def run_https():
             # logger.info("Generating https certificate...")
             print("Generating https certificate...") # TODO: remove this temporary print once logger is functional again
             cert_gen.generate_self_signed_cert(cert_file=CERT_FILE, key_file=KEY_FILE)
-
-        # Check if the certificate is valid
-        if not cert_gen.is_certificate_valid(CERT_FILE):
-            # logger.error("Invalid certificate. Cannot start https application")
-            print("Invalid certificate. Cannot start https application") # TODO: remove this temporary print once logger is functional again
-            sys.exit(1)
+        else:
+            logger.warning("Credentials already generated!")
 
         context = (CERT_FILE, KEY_FILE)
         app_restapi.run(
@@ -226,8 +229,8 @@ def run_https():
         # logger.info("HTTP server stopped by KeyboardInterrupt")
         pass
     finally:
+        logger.info("Runtime manager stopped")
         runtime_manager.stop()
-        # logger.info("Runtime manager stopped")
 
 
 if __name__ == "__main__":

webserver/config.py

@@ -4,15 +4,15 @@
 from pathlib import Path
 
 from dotenv import load_dotenv
+from logger import get_logger, LogParser
+
+logger, buffer = get_logger("logger", use_buffer=True)
 
 # Always resolve .env relative to the repo root to guarantee it is found
-ENV_PATH = Path(__file__).resolve().parent.parent / ".env"
-DB_PATH = Path(__file__).resolve().parent.parent / "restapi.db"
+ENV_PATH = Path(__file__).resolve().parent.parent / "webserver/.env"
+DB_PATH = Path(__file__).resolve().parent.parent / "webserver/restapi.db"
 BASE_DIR = os.path.abspath(os.path.dirname(__file__))
 
-# logger = logging.getLogger("logger")
-
-
 # Function to validate environment variable values
 def is_valid_env(var_name, value):
     if var_name == "SQLALCHEMY_DATABASE_URI":
@@ -35,18 +35,18 @@ def generate_env_file():
         f.write(f"PEPPER={pepper}\n")
 
     os.chmod(ENV_PATH, 0o600)
-    logger.info(f".env file created at {ENV_PATH}")
+    logger.info(".env file created at %s", ENV_PATH)
 
     # Ensure the database file exists and is writable
     # Deletion is required because new secrets will change the database saved hashes
     if os.path.exists(DB_PATH):
         os.remove(DB_PATH)
-        logger.info(f"Deleted existing database file: {DB_PATH}")
+        logger.warning("Deleted existing database file: %s", DB_PATH)
 
 
 # Load .env file
 if not os.path.isfile(ENV_PATH):
-    logger.warning(".env file not found, creating one...")
+    logger.info(".env file not found, creating one...")
     generate_env_file()
 
 load_dotenv(dotenv_path=ENV_PATH, override=False)
@@ -58,7 +58,7 @@ def generate_env_file():
         if not val or not is_valid_env(var, val):
             raise RuntimeError(f"Environment variable '{var}' is invalid or missing")
 except RuntimeError as e:
-    logger.error(f"{e}")
+    logger.error("%s", e)
     # Need to regenerate .env file and remove the database as well
     response = (
         input(
@@ -68,11 +68,11 @@ def generate_env_file():
         .lower()
     )
     if response == "y":
-        logger.info("Regenerating .env with new valid values...")
+        print("Regenerating .env with new valid values...")
         generate_env_file()
         load_dotenv(ENV_PATH)
     else:
-        logger.error("Exiting due to invalid environment configuration.")
+        print("Exiting due to invalid environment configuration.")
         exit(1)
 
 

webserver/logger/__init__.py

@@ -1,33 +1,29 @@
 import logging
-import logging.config
 from .logger import get_logger
 from .parser import LogParser
 from .bufferhandler import BufferHandler
+from .formatter import JsonFormatter
 
-__all__ = ["get_logger", "LogParser", "BufferHandler"]
+__all__ = ["get_logger", "LogParser", "BufferHandler", "JsonFormatter"]
 __version__ = "0.1"
 __author__ = "Autonomy"
 __license__ = "MIT"
 __description__ = "RestAPI interface for runtime core"
 
+# Single global buffer for all logs
+shared_buffer_handler = BufferHandler()
 
-# Configure logging once
-logging.config.dictConfig(
-    {
-        "version": 1,
-        "formatters": {
-            "default": {
-                "format": "[%(levelname)s] %(asctime)s - %(name)s - %(message)s",
-                "datefmt": "%H:%M:%S",
-            }
-        },
-        "handlers": {
-            "console": {
-                "class": "logging.StreamHandler",
-                "formatter": "default",
-                "level": "DEBUG",
-            }
-        },
-        "root": {"level": "DEBUG", "handlers": ["console"]},
-    }
-)
+formatter = JsonFormatter()
+shared_buffer_handler.setFormatter(formatter)
+
+def get_logger(name="runtime", use_buffer: bool = False):
+    """Return a logger that shares the same buffer handler."""
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.DEBUG)
+    logger.propagate = False
+
+    if use_buffer:
+        if not any(isinstance(h, BufferHandler) for h in logger.handlers):
+            logger.addHandler(shared_buffer_handler)
+
+    return logger, shared_buffer_handler

webserver/logger/bufferhandler.py

@@ -2,60 +2,98 @@
 from collections import deque
 from typing import List, Optional
 import json
-import re
-from datetime import datetime
+from datetime import datetime, timezone
+from threading import Lock
 
 
 class BufferHandler(logging.Handler):
     """
     Custom logging handler that stores log records in memory (FIFO).
     Logs are formatted using the attached formatter (JSON).
     """
+    _instance = None
+    _lock = Lock()
 
     def __init__(self, capacity: int = 1000):
         super().__init__()
         self.buffer = deque(maxlen=capacity)
 
     def emit(self, record: logging.LogRecord) -> None:
-        try:
-            self.buffer.append(self.format(record))
-        except Exception:
-            self.handleError(record)
+        with self._lock:
+            try:
+                self.buffer.append(self.format(record))
+            except Exception:
+                self.handleError(record)
+
+    def filter_logs(self, logs, level=None, min_id=None, max_id=None):
+        result = logs
+        if level is not None:
+            result = [log for log in result if log.get("level") == level]
+        if min_id is not None:
+            result = [log for log in result if log.get("id", 0) >= min_id]
+        if max_id is not None:
+            result = [log for log in result if log.get("id", 0) <= max_id]
+        return result
 
-    def get_logs(self, count: Optional[int] = None) -> List[str]:
+    def get_logs(self, count: Optional[int] = None,
+                 min_id: Optional[int] = None,
+                 level: Optional[str] = None) -> List[str]:
         """Retrieve logs from buffer."""
-        if count is None or count > len(self.buffer):
-            return list(self.buffer)
-        return list(self.buffer)[-count:]
-
-    def normalize_buffer_logs(self, buffer_records):
-        """
-        Takes a list of log strings from buffer and returns a list of clean JSON dicts.
-        """
-        result = []
-        json_extract = re.compile(r'(\{.*\})')  # match JSON inside log line
-
-        for record in buffer_records:
-            match = json_extract.search(record)
-            if not match:
-                continue
+        with self._lock:
+            filtered_logs = [json.loads(item) for item in self.buffer]
+            # json_output = json.dumps(filtered_logs, indent=2)
+            filtered_logs = self.filter_logs(filtered_logs, level=level, min_id=min_id)
+            if count is not None and count < len(filtered_logs):
+                filtered_logs = filtered_logs[-count:]
+            return filtered_logs
 
+    def normalize_timestamp_no_microseconds(self, ts: str) -> str:
+        """Normalize ISO 8601 timestamp to remove microseconds."""
+        dt = datetime.fromisoformat(ts)
+        return dt.replace(microsecond=0).strftime("%Y-%m-%dT%H:%M:%S%z")
+
+    def normalize_logs(self, json_logs: List[dict]) -> List[dict]:
+        """Normalize a list of log entries (dicts)."""
+        normalized = []
+        for data in json_logs:
             try:
-                raw_json = json.loads(match.group(1))
-                # Convert unix timestamp → readable datetime
-                ts = int(raw_json.get("timestamp", 0))
-                dt = datetime.utcfromtimestamp(ts).isoformat() + "Z"
-
-                entry = {
-                    "timestamp": dt,
-                    "level": raw_json.get("level", "INFO"),
-                    "message": raw_json.get("message", "")
-                }
-                result.append(entry)
-            except (json.JSONDecodeError, ValueError):
-                continue
+                # Normalize timestamp (convert unix timestamp → ISO 8601)
+                ts = data.get("timestamp")
 
-        return result
+                # If it's numeric (e.g., 1759843183), convert it to ISO 8601 UTC
+                if ts and str(ts).isdigit():
+                    ts_dt = datetime.fromtimestamp(int(ts), tz=timezone.utc)
+                    data["timestamp"] = ts_dt.isoformat()
+
+                # If it's ISO 8601 but has microseconds, strip them
+                if "timestamp" in data:
+                    data["timestamp"] = self.normalize_timestamp_no_microseconds(data["timestamp"])
+
+                # Ensure minimal required fields
+                data.setdefault("level", "INFO")
+                data.setdefault("message", "")
+
+                normalized.append(data)
+
+            except (json.JSONDecodeError, TypeError, ValueError) as e:
+                # If something is not JSON, safely wrap it
+                normalized.append({
+                    "id": None,
+                    "timestamp": datetime.now(timezone.utc).isoformat(),
+                    "level": "ERROR",
+                    "message": f"Malformed log: {data} ({e})",
+                })
+
+        return normalized
+
+    @classmethod
+    def get_instance(cls):
+        """Singleton accessor."""
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = cls()
+        return cls._instance
 
     def clear(self) -> None:
         self.buffer.clear()

webserver/logger/formatter.py

@@ -1,19 +1,35 @@
+from datetime import datetime, timezone
 import logging
-import time
 import json
 
+
 class JsonFormatter(logging.Formatter):
     """Format log records as JSON strings."""
+    log_id = 0
+
+    def format(self, record):        
+        msg = record.getMessage()
+        self.log_id += 1
 
-    def format(self, record: logging.LogRecord) -> str:
-        log_dict = {
-            "timestamp": str(int(record.created)),   # epoch seconds
+        # Try to detect pre-formatted JSON
+        if msg.strip().startswith("{") and msg.strip().endswith("}"):
+            try:
+                parsed = json.loads(msg)
+                # Already JSON — just make sure timestamp exists
+                if "timestamp" not in parsed:
+                    parsed["timestamp"] = datetime.now(timezone.utc).isoformat()
+                parsed["id"] = self.log_id
+                return json.dumps(parsed)
+
+            except json.JSONDecodeError:
+                pass  # continue to default formatting
+
+        # Not JSON, so create our standard JSON structure
+        log_entry = {
+            "id": self.log_id,
+            "timestamp": datetime.now(timezone.utc).isoformat(),
             "level": record.levelname,
-            "message": record.getMessage()
+            "message": msg,
         }
+        return json.dumps(log_entry)
 
-        # Include optional fields if present
-        if hasattr(record, "source"):
-            log_dict["source"] = record.source
-
-        return json.dumps(log_dict, ensure_ascii=False)