Fix Gitea issues #1 and #2, add firewall automation, forwarder telemetry, and dynamic key rotation

src/Server.py

@@ -60,14 +60,16 @@ try:
         get_machine_identifier,
         WindowsLogCollector,
         LinuxLogCollector,
-        get_log_collector
+        get_log_collector,
+        is_routine_service_noise
     )
 except ImportError:
     from log_collector import (
         get_machine_identifier,
         WindowsLogCollector,
         LinuxLogCollector,
-        get_log_collector
+        get_log_collector,
+        is_routine_service_noise
     )
 
 CONFIG_FILE_NAME = "server_config.json"
@@ -891,6 +893,42 @@ def load_or_init_config(config_path: str = CONFIG_FILE_NAME) -> Dict[str, Any]:
     return config
 
 
+def rotate_enrollment_secret(config_dict: Optional[Dict[str, Any]] = None) -> str:
+    """
+    Generates a fresh enrollment secret, invalidates the previous secret,
+    and atomically persists the change to server_config.json and database license_config.
+    """
+    cfg = config_dict if config_dict is not None else SERVER_STATE.get("config", {})
+    new_secret = secrets.token_hex(24)
+    cfg["enrollment_secret"] = new_secret
+
+    # Update config file on disk if exists
+    config_file_path = cfg.get("config_file_path") or CONFIG_FILE_NAME
+    try:
+        if os.path.exists(config_file_path):
+            with open(config_file_path, "r", encoding="utf-8") as f:
+                saved = json.load(f)
+            saved["enrollment_secret"] = new_secret
+            with open(config_file_path, "w", encoding="utf-8") as f:
+                json.dump(saved, f, indent=2)
+    except Exception as e:
+        print(f"[!] Warning: Could not write new enrollment secret to config file: {e}")
+
+    # Update database license_config table
+    db_path = cfg.get("db_path", DEFAULT_DB_FILE)
+    try:
+        conn = sqlite3.connect(db_path, timeout=5.0)
+        c = conn.cursor()
+        c.execute("UPDATE license_config SET enrollment_secret = ? WHERE id = 1", (new_secret,))
+        conn.commit()
+        conn.close()
+    except Exception as e:
+        print(f"[!] Warning: Could not update database with new enrollment secret: {e}")
+
+    SERVER_STATE["config"] = cfg
+    return new_secret
+
+
 def generate_client_profile_dict(
     server_host: str,
     server_port: int,
@@ -1079,6 +1117,18 @@ def init_db(db_path: str, enrollment_secret: Optional[str] = None, max_seats: in
             )
         """)
 
+        # Migrations for clients table
+        for col, col_def in [
+            ("metrics_json", "TEXT DEFAULT '{}'"),
+            ("cpu_percent", "REAL DEFAULT 0.0"),
+            ("memory_percent", "REAL DEFAULT 0.0"),
+            ("disk_percent_free", "REAL DEFAULT 0.0")
+        ]:
+            try:
+                conn.execute(f"ALTER TABLE clients ADD COLUMN {col} {col_def}")
+            except sqlite3.OperationalError:
+                pass
+
         # 5. Users Table
         conn.execute("""
             CREATE TABLE IF NOT EXISTS users (
@@ -1541,6 +1591,17 @@ def init_postgres_db(pg_config: Dict[str, Any]) -> Tuple[bool, str]:
                     value TEXT NOT NULL
                 );
             """)
+            # Migrations for Postgres clients table
+            for col, col_def in [
+                ("metrics_json", "TEXT DEFAULT '{}'"),
+                ("cpu_percent", "DOUBLE PRECISION DEFAULT 0.0"),
+                ("memory_percent", "DOUBLE PRECISION DEFAULT 0.0"),
+                ("disk_percent_free", "DOUBLE PRECISION DEFAULT 0.0")
+            ]:
+                try:
+                    cur.execute(f"ALTER TABLE clients ADD COLUMN IF NOT EXISTS {col} {col_def}")
+                except Exception:
+                    pass
             conn.commit()
         conn.close()
         return True, "PostgreSQL schema successfully initialized"
@@ -1682,6 +1743,11 @@ def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: i
                 server = log.get("server", client_server)
                 message = log.get("message", "")
                 os_type = log.get("os_type", "unknown")
+
+                # Suppress repetitive benign routine service start/stop cycles
+                if is_routine_service_noise(signature, message, severity):
+                    continue
+
                 fp = f"{site_name}:{server}:{signature}"
 
                 stored_message = encrypt_field_for_storage(message)
@@ -1726,6 +1792,21 @@ def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: i
 
                 processed_count += 1
 
+            # Update client node metrics if attached
+            metrics = payload.get("metrics")
+            if metrics and isinstance(metrics, dict) and client_server:
+                cpu_p = float(metrics.get("cpu_percent", 0.0))
+                mem_p = float(metrics.get("memory_percent", 0.0))
+                disk_p = float(metrics.get("disk_percent_free", 0.0))
+                try:
+                    cursor.execute("""
+                        UPDATE clients
+                        SET last_seen = %s, metrics_json = %s, cpu_percent = %s, memory_percent = %s, disk_percent_free = %s
+                        WHERE client_id = %s OR hostname = %s
+                    """, (now_iso, json.dumps(metrics), cpu_p, mem_p, disk_p, client_server, client_server))
+                except Exception:
+                    pass
+
             conn.commit()
             conn.close()
             return {
@@ -1758,6 +1839,11 @@ def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: i
         server = log.get("server", client_server)
         message = log.get("message", "")
         os_type = log.get("os_type", "unknown")
+
+        # Suppress repetitive benign routine service start/stop cycles
+        if is_routine_service_noise(signature, message, severity):
+            continue
+
         fp = f"{site_name}:{server}:{signature}"
 
         stored_message = encrypt_field_for_storage(message)
@@ -1798,10 +1884,25 @@ def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: i
                 INSERT INTO active_issues 
                 (fingerprint, site_name, server, signature, severity, message, os_type, first_seen, last_seen, run_count, status, last_run_id)
                 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-            """, (fp, site_name, server, signature, severity, message, os_type, now_iso, now_iso, 1, initial_status, run_id))
+            """, (fp, site_name, server, signature, severity, stored_message, os_type, now_iso, now_iso, 1, initial_status, run_id))
 
         processed_count += 1
 
+    # Update client node metrics if attached
+    metrics = payload.get("metrics")
+    if metrics and isinstance(metrics, dict) and client_server:
+        cpu_p = float(metrics.get("cpu_percent", 0.0))
+        mem_p = float(metrics.get("memory_percent", 0.0))
+        disk_p = float(metrics.get("disk_percent_free", 0.0))
+        try:
+            cursor.execute("""
+                UPDATE clients
+                SET last_seen = ?, metrics_json = ?, cpu_percent = ?, memory_percent = ?, disk_percent_free = ?
+                WHERE client_id = ? OR hostname = ?
+            """, (now_iso, json.dumps(metrics), cpu_p, mem_p, disk_p, client_server, client_server))
+        except Exception:
+            pass
+
     conn.commit()
     conn.close()
 
@@ -2559,8 +2660,12 @@ def export_client_profile(
     """
     Exports a standalone client profile JSON configuration bundle
     for zero-touch edge client bootstrapping.
+    Generates a fresh shared enrollment key and invalidates previous keys.
     """
     config = SERVER_STATE.get("config", {})
+    # Generate new shared enrollment secret and invalidate old key
+    rotate_enrollment_secret(config)
+
     host = server_host or config.get("tcp_host", "127.0.0.1")
     if host in ("0.0.0.0", ""):
         host = socket.gethostname()
@@ -2581,6 +2686,19 @@ def export_client_profile(
     )
 
 
+@app.post("/api/server/reset-enrollment-secret", dependencies=[Depends(require_admin_auth)])
+def reset_server_enrollment_secret():
+    """
+    Resets the shared enrollment secret key and immediately invalidates all previous keys.
+    """
+    new_secret = rotate_enrollment_secret()
+    return {
+        "status": "success",
+        "enrollment_secret": new_secret,
+        "message": "Shared enrollment key has been reset and previous keys invalidated."
+    }
+
+
 @app.get("/api/clients", dependencies=[Depends(require_admin_auth)])
 def list_clients():
     """Returns all registered clients, seat usage, and activity timestamps."""
@@ -2731,13 +2849,14 @@ def api_delete_hermes_token(req: HermesTokenRevokeRequest):
 def get_verified_anomalies_for_hermes(auth: str = Depends(require_hermes_or_admin_auth)):
     """
     Ingestion endpoint for Hermes agentic workflows.
-    Returns verified anomalies within the active 12-hour evaluation window
-    for Hermes AI direct reasoning and autonomous decision-making.
+    Returns verified anomalies within the active 12-hour evaluation window,
+    system resource context, and silent/offline forwarder alerts for Hermes AI reasoning.
     """
     db_path = SERVER_STATE.get("config", {}).get("db_path", DEFAULT_DB_FILE)
     conn = sqlite3.connect(db_path)
     cursor = conn.cursor()
 
+    # 1. Fetch verified active issues
     cursor.execute("""
         SELECT fingerprint, site_name, server, signature, severity, message, os_type, first_seen, last_seen, run_count, status
         FROM active_issues
@@ -2745,15 +2864,69 @@ def get_verified_anomalies_for_hermes(auth: str = Depends(require_hermes_or_admi
         ORDER BY last_seen DESC
     """)
     rows = cursor.fetchall()
+
+    # 2. Check for offline / silent edge nodes (Heartbeat Liveness Check)
+    now = datetime.now(timezone.utc)
+    try:
+        cursor.execute("SELECT client_id, hostname, os_type, last_seen, metrics_json, cpu_percent, memory_percent, disk_percent_free FROM clients WHERE status = 'active'")
+        client_rows = cursor.fetchall()
+    except Exception:
+        client_rows = []
     conn.close()
 
     window_hours = SERVER_STATE.get("config", {}).get("evaluation_window_hours", 12)
     report = []
+
+    # Map node telemetry and detect silent nodes
+    client_metrics_map = {}
+    for cl in client_rows:
+        cid, chost, cost, clast, cmetrics_raw, ccpu, cmem, cdisk = cl
+        try:
+            m_dict = json.loads(cmetrics_raw) if cmetrics_raw else {}
+        except Exception:
+            m_dict = {}
+        if not m_dict:
+            m_dict = {
+                "cpu_percent": ccpu or 0.0,
+                "memory_percent": cmem or 0.0,
+                "disk_percent_free": cdisk or 0.0
+            }
+        client_metrics_map[cid] = m_dict
+        if chost:
+            client_metrics_map[chost] = m_dict
+
+        # Liveness check: If active client has not checked in for > 15 minutes, report CRITICAL offline anomaly
+        if clast:
+            try:
+                clast_clean = str(clast).replace("T", " ").split("+")[0].split("Z")[0].strip()
+                last_dt = datetime.fromisoformat(clast_clean).replace(tzinfo=timezone.utc)
+                if (now - last_dt) > timedelta(minutes=15):
+                    offline_mins = int((now - last_dt).total_seconds() // 60)
+                    report.append({
+                        "fingerprint": f"offline-node-{cid}",
+                        "site": "default",
+                        "server": chost or cid,
+                        "signature": "LOGAR/NodeSilent",
+                        "severity": "CRITICAL",
+                        "message": f"Forwarder node '{chost or cid}' has gone silent! Last heartbeat was {offline_mins} minutes ago ({clast}). Potential node crash, network partition, or power failure.",
+                        "os_type": cost or "unknown",
+                        "first_seen": clast,
+                        "last_seen": clast,
+                        "consecutive_runs": 1,
+                        "evaluation_window": f"{window_hours}h",
+                        "verified": True,
+                        "status": "NODE_OFFLINE",
+                        "metrics": m_dict
+                    })
+            except Exception:
+                pass
+
     for r in rows:
+        srv_name = r[2]
         report.append({
             "fingerprint": r[0],
             "site": r[1],
-            "server": r[2],
+            "server": srv_name,
             "signature": r[3],
             "severity": r[4],
             "message": decrypt_field_from_storage(r[5]),
@@ -2763,7 +2936,8 @@ def get_verified_anomalies_for_hermes(auth: str = Depends(require_hermes_or_admi
             "consecutive_runs": r[9],
             "evaluation_window": f"{window_hours}h",
             "verified": True,
-            "status": r[10]
+            "status": r[10],
+            "metrics": client_metrics_map.get(srv_name, {})
         })
 
     return report
@@ -3847,6 +4021,7 @@ def serve_webui(request: Request = None):
                 </div>
             </div>
             <div class="header-actions">
+                <button onclick="resetEnrollmentSecret()" class="btn btn-outline" title="Reset shared enrollment secret key and invalidate old keys">🔄 Reset Shared Key</button>
                 <button onclick="openExportProfileModal()" class="btn btn-primary">📥 Export Client Profile</button>
                 <div class="user-menu-container" tabindex="0">
                     <div class="user-tag" onclick="toggleUserDropdown(event)">
@@ -3923,7 +4098,10 @@ def serve_webui(request: Request = None):
             <div class="card">
                 <div class="card-header">
                     <span class="card-title" style="font-size: 15px;">Enrolled Forwarders & Seat Management</span>
-                    <button onclick="openExportProfileModal()" class="btn btn-accent" style="padding:6px 12px; font-size:12px;">+ Export New Client Profile</button>
+                    <div style="display: flex; gap: 8px;">
+                        <button onclick="resetEnrollmentSecret()" class="btn btn-outline" style="padding:6px 12px; font-size:12px;" title="Reset shared enrollment secret key">🔄 Reset Shared Key</button>
+                        <button onclick="openExportProfileModal()" class="btn btn-accent" style="padding:6px 12px; font-size:12px;">+ Export New Client Profile</button>
+                    </div>
                 </div>
                 <div style="overflow-x: auto;">
                     <table>
@@ -4101,7 +4279,10 @@ def serve_webui(request: Request = None):
             </div>
 
             <div style="background: #0f172a; padding: 12px; border-radius: 8px; border: 1px solid var(--border); margin-bottom: 16px;">
-                <div style="font-size: 11px; font-weight: 700; text-transform: uppercase; color: var(--text-muted); margin-bottom: 6px;">Edge Client Quick Deploy Command</div>
+                <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 6px;">
+                    <div style="font-size: 11px; font-weight: 700; text-transform: uppercase; color: var(--text-muted);">Edge Client Quick Deploy Command</div>
+                    <button onclick="resetEnrollmentSecret()" class="btn btn-outline" style="padding: 2px 8px; font-size: 10px;" title="Reset shared enrollment secret key">🔄 Reset Key</button>
+                </div>
                 <pre id="quickDeployCmd" style="font-family: 'JetBrains Mono', monospace; font-size: 11px; color: #38bdf8; word-break: break-all; white-space: pre-wrap;">sudo ./compilation/install_linux_client.sh "http://{detected_host}:{config.get('hermes_port', 8443)}" "{config.get('enrollment_secret')}"</pre>
             </div>
 
@@ -4323,6 +4504,29 @@ def serve_webui(request: Request = None):
             }}
         }}
 
+        async function resetEnrollmentSecret() {{
+            if (!confirm('Are you sure you want to reset the shared client enrollment key? Any un-enrolled forwarders using previous configuration keys will need updated configurations.')) return;
+            try {{
+                const res = await fetch('/api/server/reset-enrollment-secret', {{
+                    method: 'POST',
+                    headers: {{ 'Content-Type': 'application/json' }}
+                }});
+                const data = await res.json();
+                if (res.ok) {{
+                    showAlert('Shared enrollment key reset successfully! Old keys are now invalidated.');
+                    const deployEl = document.getElementById('quickDeployCmd');
+                    if (deployEl && data.enrollment_secret) {{
+                        const host = document.getElementById('exportHost') ? document.getElementById('exportHost').value.trim() : '{detected_host}';
+                        deployEl.textContent = `sudo ./compilation/install_linux_client.sh "http://${{host}}:{config.get('hermes_port', 8443)}" "${{data.enrollment_secret}}"`;
+                    }}
+                }} else {{
+                    alert('Reset failed: ' + (data.detail || 'Unknown error'));
+                }}
+            }} catch (err) {{
+                alert('Reset error: ' + err.message);
+            }}
+        }}
+
         function downloadClientProfile() {{
             const host = document.getElementById('exportHost').value.trim();
             const port = document.getElementById('exportPort').value.trim();

src/client_transport.py

@@ -23,7 +23,8 @@ from log_collector import (
     get_machine_identifier,
     is_path_writable,
     get_default_data_dir,
-    resolve_writable_path
+    resolve_writable_path,
+    collect_system_metrics
 )
 
 CONFIG_FILE_NAME = "client_config.json"
@@ -247,7 +248,8 @@ def send_encrypted_logs_over_socket(
             payload = {
                 "server": actual_machine_id,
                 "timestamp": datetime.now(timezone.utc).isoformat(),
-                "logs": logs
+                "logs": logs,
+                "metrics": collect_system_metrics()
             }
             payload_bytes = json.dumps(payload).encode("utf-8")
             frame = struct.pack(">I", len(payload_bytes)) + payload_bytes
@@ -283,7 +285,8 @@ def send_encrypted_logs_over_socket(
     payload = {
         "server": actual_machine_id,
         "timestamp": datetime.now(timezone.utc).isoformat(),
-        "logs": logs
+        "logs": logs,
+        "metrics": collect_system_metrics()
     }
     payload_json = json.dumps(payload)
     pgp_msg = pgpy.PGPMessage.new(payload_json)

src/log_collector.py

@@ -8,6 +8,8 @@ import os
 import sys
 import json
 import socket
+import shutil
+import ctypes
 import subprocess
 import warnings
 from abc import ABC, abstractmethod
@@ -26,6 +28,141 @@ CONFIG_FILE_NAME = "client_config.json"
 STATE_FILE_NAME = "client_state.json"
 
 
+def collect_system_metrics() -> Dict[str, Any]:
+    """
+    Collects cross-platform host health and resource telemetry without requiring external wheels.
+    Returns:
+        {
+            "cpu_percent": float,
+            "memory_percent": float,
+            "memory_total_mb": int,
+            "memory_free_mb": int,
+            "disk_percent_free": float,
+            "disk_total_gb": float,
+            "disk_free_gb": float
+        }
+    """
+    metrics = {
+        "cpu_percent": 0.0,
+        "memory_percent": 0.0,
+        "memory_total_mb": 0,
+        "memory_free_mb": 0,
+        "disk_percent_free": 0.0,
+        "disk_total_gb": 0.0,
+        "disk_free_gb": 0.0
+    }
+    # 1. Disk metrics
+    try:
+        path = "C:\\" if sys.platform == "win32" else "/"
+        total, used, free = shutil.disk_usage(path)
+        metrics["disk_total_gb"] = round(total / (1024 ** 3), 2)
+        metrics["disk_free_gb"] = round(free / (1024 ** 3), 2)
+        if total > 0:
+            metrics["disk_percent_free"] = round((free / total) * 100, 1)
+    except Exception:
+        pass
+
+    # 2. Memory metrics
+    try:
+        if sys.platform == "win32":
+            class MEMORYSTATUSEX(ctypes.Structure):
+                _fields_ = [
+                    ("dwLength", ctypes.c_ulong),
+                    ("dwMemoryLoad", ctypes.c_ulong),
+                    ("ullTotalPhys", ctypes.c_ulonglong),
+                    ("ullAvailPhys", ctypes.c_ulonglong),
+                    ("ullTotalPageFile", ctypes.c_ulonglong),
+                    ("ullAvailPageFile", ctypes.c_ulonglong),
+                    ("ullTotalVirtual", ctypes.c_ulonglong),
+                    ("ullAvailVirtual", ctypes.c_ulonglong),
+                    ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
+                ]
+            stat = MEMORYSTATUSEX()
+            stat.dwLength = ctypes.sizeof(MEMORYSTATUSEX)
+            if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)):
+                metrics["memory_percent"] = float(stat.dwMemoryLoad)
+                metrics["memory_total_mb"] = int(stat.ullTotalPhys / (1024 * 1024))
+                metrics["memory_free_mb"] = int(stat.ullAvailPhys / (1024 * 1024))
+        else:
+            # Linux: read /proc/meminfo
+            if os.path.exists("/proc/meminfo"):
+                mem_total = 0
+                mem_avail = 0
+                with open("/proc/meminfo", "r", encoding="utf-8") as f:
+                    for line in f:
+                        parts = line.split(":")
+                        if len(parts) == 2:
+                            k, v = parts[0].strip(), parts[1].strip().split()[0]
+                            if k == "MemTotal":
+                                mem_total = int(v)
+                            elif k in ("MemAvailable", "MemFree") and mem_avail == 0:
+                                mem_avail = int(v)
+                if mem_total > 0:
+                    metrics["memory_total_mb"] = mem_total // 1024
+                    metrics["memory_free_mb"] = mem_avail // 1024
+                    metrics["memory_percent"] = round(((mem_total - mem_avail) / mem_total) * 100, 1)
+    except Exception:
+        pass
+
+    # 3. CPU metrics
+    try:
+        if hasattr(os, "getloadavg"):
+            load1, _, _ = os.getloadavg()
+            cpu_count = os.cpu_count() or 1
+            metrics["cpu_percent"] = round(min(100.0, (load1 / cpu_count) * 100), 1)
+        else:
+            try:
+                import psutil
+                metrics["cpu_percent"] = float(psutil.cpu_percent(interval=None))
+            except ImportError:
+                metrics["cpu_percent"] = 0.0
+    except Exception:
+        pass
+
+    return metrics
+
+
+def is_routine_service_noise(signature: str, message: str, severity: str) -> bool:
+    """
+    Identifies high-frequency benign background service start/stop cycles
+    (e.g., sppsvc, edgeupdate, trustedinstaller, wuauserv, backgroundTaskHost)
+    to prevent false-positive anomaly fatigue and consecutive_run metric bloat.
+    """
+    if str(severity).upper() != "INFO":
+        return False
+    sig_low = (signature or "").lower()
+    msg_low = (message or "").lower()
+    routine_services = [
+        "sppsvc",
+        "edgeupdate",
+        "trustedinstaller",
+        "wuauserv",
+        "backgroundtaskhost",
+        "service control manager",
+        "systemd",
+        "cron"
+    ]
+    is_routine_svc = any(s in sig_low or s in msg_low for s in routine_services)
+    if is_routine_svc:
+        lifecycle_phrases = [
+            "entered the running state",
+            "entered the stopped state",
+            "entered the start pending state",
+            "entered the stop pending state",
+            "started successfully",
+            "stopped successfully",
+            "service stopped",
+            "service started",
+            "session opened",
+            "session closed",
+            "clean",
+            "reloaded"
+        ]
+        if any(p in msg_low for p in lifecycle_phrases):
+            return True
+    return False
+
+
 def get_machine_identifier() -> str:
     """
     Returns the fully qualified hostname of the machine sending/collecting logs.