[RTOP-76] runtime logs parser

.vscode/settings.json

@@ -5,7 +5,8 @@
         "dlfcn.h": "c",
         "utils.h": "c",
         "plc_state_manager.h": "c",
-        "unix_socket.h": "c"
+        "unix_socket.h": "c",
+        "plugin_driver.h": "c"
     },
     "editor.rulers": [
         80

webserver/app.py

@@ -1,4 +1,3 @@
-import logging
 import os
 import ssl
 from pathlib import Path
@@ -28,12 +27,15 @@
     MAX_FILE_SIZE
 )
 
+# from logger import get_logger, LogParser
+
+
 app = flask.Flask(__name__)
 app.secret_key = str(os.urandom(16))
 login_manager = flask_login.LoginManager()
 login_manager.init_app(app)
 
-logger = logging.getLogger(__name__)
+# logger = get_logger(use_buffer=True)
 
 runtime_manager = RuntimeManager(
     runtime_path="./build/plc_main",
@@ -97,7 +99,7 @@ def restapi_callback_get(argument: str, data: dict) -> dict:
     """
     Dispatch GET callbacks by argument.
     """
-    logger.debug("GET | Received argument: %s, data: %s", argument, data)
+    # logger.debug("GET | Received argument: %s, data: %s", argument, data)
     handler = GET_HANDLERS.get(argument)
     if handler:
         return handler(data)
@@ -166,7 +168,7 @@ def restapi_callback_post(argument: str, data: dict) -> dict:
     """
     Dispatch POST callbacks by argument.
     """
-    logger.debug("POST | Received argument: %s, data: %s", argument, data)
+    # logger.debug("POST | Received argument: %s, data: %s", argument, data)
     handler = POST_HANDLERS.get(argument)
 
     if not handler:
@@ -184,9 +186,10 @@ def run_https():
         try:
             db.create_all()
             db.session.commit()
-            logger.info("Database tables created successfully.")
+            # logger.info("Database tables created successfully.")
         except Exception as e:
-            logger.error("Error creating database tables: %s", e)
+            # logger.error("Error creating database tables: %s", e)
+            pass
 
     try:
         cert_gen = CertGen(hostname=HOSTNAME, ip_addresses=["127.0.0.1"])
@@ -208,14 +211,17 @@ def run_https():
         )
 
     except FileNotFoundError as e:
-        logger.error("Could not find SSL credentials! %s", e)
+        # logger.error("Could not find SSL credentials! %s", e)
+        pass
     except ssl.SSLError as e:
-        logger.error("SSL credentials FAIL! %s", e)
+        # logger.error("SSL credentials FAIL! %s", e)
+        pass
     except KeyboardInterrupt:
-        logger.info("HTTP server stopped by KeyboardInterrupt")
+        # logger.info("HTTP server stopped by KeyboardInterrupt")
+        pass
     finally:
         runtime_manager.stop()
-        logger.info("Runtime manager stopped")
+        # logger.info("Runtime manager stopped")
 
 
 if __name__ == "__main__":

webserver/config.py

@@ -1,4 +1,3 @@
-import logging
 import os
 import re
 import secrets
@@ -11,12 +10,7 @@
 DB_PATH = Path(__file__).resolve().parent.parent / "restapi.db"
 BASE_DIR = os.path.abspath(os.path.dirname(__file__))
 
-logger = logging.getLogger(__name__)
-logging.basicConfig(
-    level=logging.DEBUG,  # Minimum level to capture
-    format="[%(levelname)s] %(asctime)s - %(message)s",
-    datefmt="%H:%M:%S",
-)
+# logger = logging.getLogger("logger")
 
 
 # Function to validate environment variable values

webserver/credentials.py

@@ -1,15 +1,14 @@
 import datetime
 import ipaddress
 import os
-import logging
 
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes, serialization
 from cryptography.hazmat.primitives.asymmetric import rsa
 from cryptography.x509.oid import NameOID
 
-logger = logging.getLogger(__name__)
+# logger = logging.getLogger("logger")
 
 
 class CertGen:
@@ -40,7 +39,7 @@ def generate_key(self):
         )
 
     def generate_self_signed_cert(self, cert_file, key_file):
-        logger.debug("Generating self-signed certificate for %s...", self.hostname)
+        # logger.debug("Generating self-signed certificate for %s...", self.hostname)
 
         self.generate_key()
 
@@ -72,13 +71,13 @@ def generate_self_signed_cert(self, cert_file, key_file):
                     serialization.NoEncryption(),
                 )
             )
-        logger.debug("Certificate saved to %s", cert_file)
-        logger.debug("Private key saved to %s", key_file)
+        # logger.debug("Certificate saved to %s", cert_file)
+        # logger.debug("Private key saved to %s", key_file)
 
     def is_certificate_valid(self, cert_file):
         """Check if the certificate is valid."""
         if not os.path.exists(cert_file):
-            logger.warning("Certificate file not found: %s", cert_file)
+            # logger.warning("Certificate file not found: %s", cert_file)
             return False
 
         try:
@@ -90,15 +89,15 @@ def is_certificate_valid(self, cert_file):
             now = datetime.datetime.now(datetime.timezone.utc)
 
             if now < cert.not_valid_before_utc:
-                logger.warning("Certificate is not yet valid. Valid from: %s", cert.not_valid_before_utc)
+                # logger.warning("Certificate is not yet valid. Valid from: %s", cert.not_valid_before_utc)
                 return False
             if now > cert.not_valid_after_utc:
-                logger.warning("Certificate has expired. Expired on: %s", cert.not_valid_after_utc)
+                # logger.warning("Certificate has expired. Expired on: %s", cert.not_valid_after_utc)
                 return False
 
-            logger.info("Certificate is valid. Expires on: %s", cert.not_valid_after_utc)
+            # logger.info("Certificate is valid. Expires on: %s", cert.not_valid_after_utc)
             return True
 
         except Exception as e:
-            logger.error("Error loading or parsing certificate: %s", e)
+            # logger.error("Error loading or parsing certificate: %s", e)
             return False

webserver/__init__.py → webserver/logger/__init__.py

@@ -1,6 +1,10 @@
 import logging
 import logging.config
+from .logger import get_logger
+from .parser import LogParser
+from .bufferhandler import BufferHandler
 
+__all__ = ["get_logger", "LogParser", "BufferHandler"]
 __version__ = "0.1"
 __author__ = "Autonomy"
 __license__ = "MIT"

webserver/logger/bufferhandler.py

@@ -0,0 +1,64 @@
+import logging
+from collections import deque
+from typing import List, Optional
+import json
+import re
+from datetime import datetime
+
+
+class BufferHandler(logging.Handler):
+    """
+    Custom logging handler that stores log records in memory (FIFO).
+    Logs are formatted using the attached formatter (JSON).
+    """
+
+    def __init__(self, capacity: int = 1000):
+        super().__init__()
+        self.buffer = deque(maxlen=capacity)
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            self.buffer.append(self.format(record))
+        except Exception:
+            self.handleError(record)
+
+    def get_logs(self, count: Optional[int] = None) -> List[str]:
+        """Retrieve logs from buffer."""
+        if count is None or count > len(self.buffer):
+            return list(self.buffer)
+        return list(self.buffer)[-count:]
+
+    def normalize_buffer_logs(self, buffer_records):
+        """
+        Takes a list of log strings from buffer and returns a list of clean JSON dicts.
+        """
+        result = []
+        json_extract = re.compile(r'(\{.*\})')  # match JSON inside log line
+
+        for record in buffer_records:
+            match = json_extract.search(record)
+            if not match:
+                continue
+
+            try:
+                raw_json = json.loads(match.group(1))
+                # Convert unix timestamp → readable datetime
+                ts = int(raw_json.get("timestamp", 0))
+                dt = datetime.utcfromtimestamp(ts).isoformat() + "Z"
+
+                entry = {
+                    "timestamp": dt,
+                    "level": raw_json.get("level", "INFO"),
+                    "message": raw_json.get("message", "")
+                }
+                result.append(entry)
+            except (json.JSONDecodeError, ValueError):
+                continue
+
+        return result
+
+    def clear(self) -> None:
+        self.buffer.clear()
+
+    def __len__(self):
+        return len(self.buffer)

webserver/logger/formatter.py

@@ -0,0 +1,19 @@
+import logging
+import time
+import json
+
+class JsonFormatter(logging.Formatter):
+    """Format log records as JSON strings."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        log_dict = {
+            "timestamp": str(int(record.created)),   # epoch seconds
+            "level": record.levelname,
+            "message": record.getMessage()
+        }
+
+        # Include optional fields if present
+        if hasattr(record, "source"):
+            log_dict["source"] = record.source
+
+        return json.dumps(log_dict, ensure_ascii=False)

webserver/logger/logger.py

@@ -0,0 +1,38 @@
+import logging
+from .formatter import JsonFormatter
+from .bufferhandler import BufferHandler
+
+
+def get_logger(name: str = "logger", 
+               level: int = logging.INFO, 
+               use_buffer: bool = False):
+    """Return a logger instance with custom formatting."""
+
+    collector_logger = logging.getLogger(name)
+    collector_logger.setLevel(logging.DEBUG)
+
+    handler = logging.StreamHandler()
+    handler.setFormatter(JsonFormatter())
+    collector_logger.addHandler(handler)
+
+    buffer_handler = None
+
+    if use_buffer:
+        # Use buffer handler for log messages
+        buffer_handler = BufferHandler()
+        buffer_handler.setFormatter(
+            logging.Formatter("%(asctime)s - %(levelname)s - %(message)s"))
+        collector_logger.addHandler(buffer_handler)
+
+    if use_buffer:
+        # Find buffer handler again if it already exists
+        if buffer_handler is None:
+            for h in collector_logger.handlers:
+                if isinstance(h, BufferHandler):
+                    buffer_handler = h
+                    break
+        return collector_logger, buffer_handler
+    else:
+        return collector_logger, None
+
+    # return collector_logger

webserver/logger/parser.py

@@ -0,0 +1,74 @@
+# logger/parser.py
+import logging
+import re
+import time
+import json
+
+LOG_PATTERN = re.compile(r'^\[(?P<level>\w+)\]\s*(?P<message>.*)$')
+
+LEVEL_MAP = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR,
+    "CRITICAL": logging.CRITICAL,
+}
+
+
+class LogParser:
+    def __init__(self, collector_logger: logging.Logger):
+        self.collector_logger = collector_logger
+
+    def parse_and_log(self, line: str):
+        """Parse incoming log line and re-log it in normalized JSON format."""
+        sline = line.strip()
+        if not sline:
+            return
+
+        timestamp = int(time.time())
+        level_name = "INFO"
+        level = logging.INFO
+        message = sline
+
+        # Case 1: JSON log already
+        try:
+            parsed = json.loads(sline)
+            if isinstance(parsed, dict) and "message" in parsed:
+                # Preserve incoming JSON fields, but ensure timestamp is present
+                parsed.setdefault("timestamp", str(timestamp))
+                level_name = parsed.get("level", "INFO")
+                level = LEVEL_MAP.get(level_name, logging.INFO)
+                log_entry = parsed
+            else:
+                raise ValueError("Not a valid log JSON dict")
+        except (json.JSONDecodeError, ValueError):
+            # Case 2: Regex log like "[INFO] Something"
+            match = LOG_PATTERN.match(sline)
+            if match:
+                level_name = match["level"]
+                level = LEVEL_MAP.get(level_name, logging.INFO)
+                message = match["message"]
+            else:
+                message = sline
+
+            log_entry = {
+                "timestamp": str(timestamp),
+                "level": level_name,
+                "message": message
+            }
+
+        # Create final JSON string
+        json_log = json.dumps(log_entry, ensure_ascii=False)
+
+        # Push into Python logging
+        record = self.collector_logger.makeRecord(
+            name="external",
+            level=level,
+            fn="",
+            lno=0,
+            msg=json_log,
+            args=(),
+            exc_info=None
+        )
+        record.source = "external"
+        self.collector_logger.handle(record)