Fix Gitea issues #1 and #2, add firewall automation, forwarder telemetry, and dynamic key rotation

RELEASE_NOTES.md

@@ -1,5 +1,22 @@
 # LOGAR Release Notes
 
+## [v0.3.5.2] - 2026-10-02
+
+### 🛡️ Windows Defender Firewall Automation (Fixes Issue #1)
+- **Automated Port Exemptions**: Inno Setup (`installer_server.iss`) and NSIS (`installer_server.nsi`) server installers automatically configure inbound Windows Defender Firewall rules for WebUI (`TCP 8443`) and mTLS log ingestion (`TCP 9443`).
+- **Clean Uninstallation**: Uninstallers automatically remove the created firewall rules cleanly upon server decommissioning.
+
+### 📊 Forwarder Liveness, Silent Server Detection & System Telemetry (Fixes Issue #2)
+- **Silent Node & Crash Detection**: Resolved the "Silent Server Down Bug" by tracking forwarder heartbeats (`last_seen`). Hermes reporting (`GET /api/hermes/report`) automatically flags edge nodes that have gone silent for >15 minutes as `CRITICAL` `NODE_OFFLINE` alerts.
+- **Troubleshooting System Telemetry**: Cross-platform system telemetry metrics (CPU utilization %, Memory load %, and Free Disk space %) are collected and attached to client heartbeats and Hermes anomaly records without external dependencies.
+- **Benign Service Noise Suppression**: Filtered routine repetitive Windows and Linux background service lifecycle events (e.g., `sppsvc`, `edgeupdate`, `trustedinstaller`, `wuauserv`, `backgroundTaskHost`) from false-positive anomaly alert fatigue.
+
+### 🔑 Dynamic Enrollment Key Rotation & WebUI Key Management
+- **Automatic Key Invalidation**: Clicking "Download Client Config" (`GET /api/client/export-profile`) automatically generates a fresh enrollment secret and invalidates previous secrets, preventing reuse of leaked keys.
+- **WebUI Reset Shared Key**: Added "🔄 Reset Shared Key" controls in the WebUI header, Fleet tab, and Export Modal, backed by `POST /api/server/reset-enrollment-secret` to rotate keys and update quick-deploy command snippets on demand.
+
+---
+
 ## [v0.3.5.1] - 2026-09-30
 
 ### 🗄️ Multi-Engine Storage Architecture

compilation/installer_server.iss

@@ -45,9 +45,13 @@ Filename: "{app}\nssm.exe"; Parameters: "set LOGAR_Server AppDirectory ""{app}""
 Filename: "{app}\nssm.exe"; Parameters: "set LOGAR_Server AppStdout ""{code:GetDataDir}\server.log"""; Flags: runhidden
 Filename: "{app}\nssm.exe"; Parameters: "set LOGAR_Server AppStderr ""{code:GetDataDir}\server_err.log"""; Flags: runhidden
 Filename: "{app}\nssm.exe"; Parameters: "set LOGAR_Server AppParameters ""--config """"{code:GetDataDir}\server_config.json"""""""; Flags: runhidden
+Filename: "netsh.exe"; Parameters: "advfirewall firewall add rule name=""LOGAR Server WebUI (8443)"" dir=in action=allow protocol=TCP localport=8443"; Flags: runhidden
+Filename: "netsh.exe"; Parameters: "advfirewall firewall add rule name=""LOGAR Server Ingestion (9443)"" dir=in action=allow protocol=TCP localport=9443"; Flags: runhidden
 Filename: "{app}\nssm.exe"; Parameters: "start LOGAR_Server"; Flags: runhidden
 
 [UninstallRun]
+Filename: "netsh.exe"; Parameters: "advfirewall firewall delete rule name=""LOGAR Server WebUI (8443)"""; Flags: runhidden
+Filename: "netsh.exe"; Parameters: "advfirewall firewall delete rule name=""LOGAR Server Ingestion (9443)"""; Flags: runhidden
 Filename: "{app}\nssm.exe"; Parameters: "stop LOGAR_Server"; Flags: runhidden
 Filename: "{app}\nssm.exe"; Parameters: "remove LOGAR_Server confirm"; Flags: runhidden
 

compilation/installer_server.nsi

@@ -134,6 +134,10 @@ Section "Install"
     CreateDirectory "$DATADIR"
     ExecWait 'icacls "$DATADIR" /grant "*S-1-5-32-545:(OI)(CI)M" /T /Q'
 
+    # Configure Windows Defender Firewall inbound rules for WebUI (8443) and Ingestion (9443)
+    ExecWait 'netsh advfirewall firewall add rule name="LOGAR Server WebUI (8443)" dir=in action=allow protocol=TCP localport=8443'
+    ExecWait 'netsh advfirewall firewall add rule name="LOGAR Server Ingestion (9443)" dir=in action=allow protocol=TCP localport=9443'
+
     # Install and configure service via NSSM
     ExecWait '"$INSTDIR\nssm.exe" install LOGAR_Server "$INSTDIR\Server.exe"'
     ExecWait '"$INSTDIR\nssm.exe" set LOGAR_Server AppDirectory "$INSTDIR"'
@@ -154,7 +158,9 @@ Section "Uninstall"
         StrCpy $DATADIR "$APPDATA\LOGAR-Server"
     ${EndIf}
 
-    # 1. Stop and remove service
+    # 1. Stop and remove service & clean firewall rules
+    ExecWait 'netsh advfirewall firewall delete rule name="LOGAR Server WebUI (8443)"'
+    ExecWait 'netsh advfirewall firewall delete rule name="LOGAR Server Ingestion (9443)"'
     ExecWait 'net stop LOGAR_Server'
     ExecWait 'sc stop LOGAR_Server'
     ${If} ${FileExists} "$INSTDIR\nssm.exe"

src/Server.py

@@ -60,14 +60,16 @@ try:
         get_machine_identifier,
         WindowsLogCollector,
         LinuxLogCollector,
-        get_log_collector
+        get_log_collector,
+        is_routine_service_noise
     )
 except ImportError:
     from log_collector import (
         get_machine_identifier,
         WindowsLogCollector,
         LinuxLogCollector,
-        get_log_collector
+        get_log_collector,
+        is_routine_service_noise
     )
 
 CONFIG_FILE_NAME = "server_config.json"
@@ -891,6 +893,42 @@ def load_or_init_config(config_path: str = CONFIG_FILE_NAME) -> Dict[str, Any]:
     return config
 
 
+def rotate_enrollment_secret(config_dict: Optional[Dict[str, Any]] = None) -> str:
+    """
+    Generates a fresh enrollment secret, invalidates the previous secret,
+    and atomically persists the change to server_config.json and database license_config.
+    """
+    cfg = config_dict if config_dict is not None else SERVER_STATE.get("config", {})
+    new_secret = secrets.token_hex(24)
+    cfg["enrollment_secret"] = new_secret
+
+    # Update config file on disk if exists
+    config_file_path = cfg.get("config_file_path") or CONFIG_FILE_NAME
+    try:
+        if os.path.exists(config_file_path):
+            with open(config_file_path, "r", encoding="utf-8") as f:
+                saved = json.load(f)
+            saved["enrollment_secret"] = new_secret
+            with open(config_file_path, "w", encoding="utf-8") as f:
+                json.dump(saved, f, indent=2)
+    except Exception as e:
+        print(f"[!] Warning: Could not write new enrollment secret to config file: {e}")
+
+    # Update database license_config table
+    db_path = cfg.get("db_path", DEFAULT_DB_FILE)
+    try:
+        conn = sqlite3.connect(db_path, timeout=5.0)
+        c = conn.cursor()
+        c.execute("UPDATE license_config SET enrollment_secret = ? WHERE id = 1", (new_secret,))
+        conn.commit()
+        conn.close()
+    except Exception as e:
+        print(f"[!] Warning: Could not update database with new enrollment secret: {e}")
+
+    SERVER_STATE["config"] = cfg
+    return new_secret
+
+
 def generate_client_profile_dict(
     server_host: str,
     server_port: int,
@@ -1079,6 +1117,18 @@ def init_db(db_path: str, enrollment_secret: Optional[str] = None, max_seats: in
             )
         """)
 
+        # Migrations for clients table
+        for col, col_def in [
+            ("metrics_json", "TEXT DEFAULT '{}'"),
+            ("cpu_percent", "REAL DEFAULT 0.0"),
+            ("memory_percent", "REAL DEFAULT 0.0"),
+            ("disk_percent_free", "REAL DEFAULT 0.0")
+        ]:
+            try:
+                conn.execute(f"ALTER TABLE clients ADD COLUMN {col} {col_def}")
+            except sqlite3.OperationalError:
+                pass
+
         # 5. Users Table
         conn.execute("""
             CREATE TABLE IF NOT EXISTS users (
@@ -1541,6 +1591,17 @@ def init_postgres_db(pg_config: Dict[str, Any]) -> Tuple[bool, str]:
                     value TEXT NOT NULL
                 );
             """)
+            # Migrations for Postgres clients table
+            for col, col_def in [
+                ("metrics_json", "TEXT DEFAULT '{}'"),
+                ("cpu_percent", "DOUBLE PRECISION DEFAULT 0.0"),
+                ("memory_percent", "DOUBLE PRECISION DEFAULT 0.0"),
+                ("disk_percent_free", "DOUBLE PRECISION DEFAULT 0.0")
+            ]:
+                try:
+                    cur.execute(f"ALTER TABLE clients ADD COLUMN IF NOT EXISTS {col} {col_def}")
+                except Exception:
+                    pass
             conn.commit()
         conn.close()
         return True, "PostgreSQL schema successfully initialized"
@@ -1682,6 +1743,11 @@ def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: i
                 server = log.get("server", client_server)
                 message = log.get("message", "")
                 os_type = log.get("os_type", "unknown")
+
+                # Suppress repetitive benign routine service start/stop cycles
+                if is_routine_service_noise(signature, message, severity):
+                    continue
+
                 fp = f"{site_name}:{server}:{signature}"
 
                 stored_message = encrypt_field_for_storage(message)
@@ -1726,6 +1792,21 @@ def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: i
 
                 processed_count += 1
 
+            # Update client node metrics if attached
+            metrics = payload.get("metrics")
+            if metrics and isinstance(metrics, dict) and client_server:
+                cpu_p = float(metrics.get("cpu_percent", 0.0))
+                mem_p = float(metrics.get("memory_percent", 0.0))
+                disk_p = float(metrics.get("disk_percent_free", 0.0))
+                try:
+                    cursor.execute("""
+                        UPDATE clients
+                        SET last_seen = %s, metrics_json = %s, cpu_percent = %s, memory_percent = %s, disk_percent_free = %s
+                        WHERE client_id = %s OR hostname = %s
+                    """, (now_iso, json.dumps(metrics), cpu_p, mem_p, disk_p, client_server, client_server))
+                except Exception:
+                    pass
+
             conn.commit()
             conn.close()
             return {
@@ -1758,6 +1839,11 @@ def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: i
         server = log.get("server", client_server)
         message = log.get("message", "")
         os_type = log.get("os_type", "unknown")
+
+        # Suppress repetitive benign routine service start/stop cycles
+        if is_routine_service_noise(signature, message, severity):
+            continue
+
         fp = f"{site_name}:{server}:{signature}"
 
         stored_message = encrypt_field_for_storage(message)
@@ -1798,10 +1884,25 @@ def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: i
                 INSERT INTO active_issues 
                 (fingerprint, site_name, server, signature, severity, message, os_type, first_seen, last_seen, run_count, status, last_run_id)
                 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-            """, (fp, site_name, server, signature, severity, message, os_type, now_iso, now_iso, 1, initial_status, run_id))
+            """, (fp, site_name, server, signature, severity, stored_message, os_type, now_iso, now_iso, 1, initial_status, run_id))
 
         processed_count += 1
 
+    # Update client node metrics if attached
+    metrics = payload.get("metrics")
+    if metrics and isinstance(metrics, dict) and client_server:
+        cpu_p = float(metrics.get("cpu_percent", 0.0))
+        mem_p = float(metrics.get("memory_percent", 0.0))
+        disk_p = float(metrics.get("disk_percent_free", 0.0))
+        try:
+            cursor.execute("""
+                UPDATE clients
+                SET last_seen = ?, metrics_json = ?, cpu_percent = ?, memory_percent = ?, disk_percent_free = ?
+                WHERE client_id = ? OR hostname = ?
+            """, (now_iso, json.dumps(metrics), cpu_p, mem_p, disk_p, client_server, client_server))
+        except Exception:
+            pass
+
     conn.commit()
     conn.close()
 
@@ -2559,8 +2660,12 @@ def export_client_profile(
     """
     Exports a standalone client profile JSON configuration bundle
     for zero-touch edge client bootstrapping.
+    Generates a fresh shared enrollment key and invalidates previous keys.
     """
     config = SERVER_STATE.get("config", {})
+    # Generate new shared enrollment secret and invalidate old key
+    rotate_enrollment_secret(config)
+
     host = server_host or config.get("tcp_host", "127.0.0.1")
     if host in ("0.0.0.0", ""):
         host = socket.gethostname()
@@ -2581,6 +2686,19 @@ def export_client_profile(
     )
 
 
+@app.post("/api/server/reset-enrollment-secret", dependencies=[Depends(require_admin_auth)])
+def reset_server_enrollment_secret():
+    """
+    Resets the shared enrollment secret key and immediately invalidates all previous keys.
+    """
+    new_secret = rotate_enrollment_secret()
+    return {
+        "status": "success",
+        "enrollment_secret": new_secret,
+        "message": "Shared enrollment key has been reset and previous keys invalidated."
+    }
+
+
 @app.get("/api/clients", dependencies=[Depends(require_admin_auth)])
 def list_clients():
     """Returns all registered clients, seat usage, and activity timestamps."""
@@ -2731,13 +2849,14 @@ def api_delete_hermes_token(req: HermesTokenRevokeRequest):
 def get_verified_anomalies_for_hermes(auth: str = Depends(require_hermes_or_admin_auth)):
     """
     Ingestion endpoint for Hermes agentic workflows.
-    Returns verified anomalies within the active 12-hour evaluation window
-    for Hermes AI direct reasoning and autonomous decision-making.
+    Returns verified anomalies within the active 12-hour evaluation window,
+    system resource context, and silent/offline forwarder alerts for Hermes AI reasoning.
     """
     db_path = SERVER_STATE.get("config", {}).get("db_path", DEFAULT_DB_FILE)
     conn = sqlite3.connect(db_path)
     cursor = conn.cursor()
 
+    # 1. Fetch verified active issues
     cursor.execute("""
         SELECT fingerprint, site_name, server, signature, severity, message, os_type, first_seen, last_seen, run_count, status
         FROM active_issues
@@ -2745,15 +2864,69 @@ def get_verified_anomalies_for_hermes(auth: str = Depends(require_hermes_or_admi
         ORDER BY last_seen DESC
     """)
     rows = cursor.fetchall()
+
+    # 2. Check for offline / silent edge nodes (Heartbeat Liveness Check)
+    now = datetime.now(timezone.utc)
+    try:
+        cursor.execute("SELECT client_id, hostname, os_type, last_seen, metrics_json, cpu_percent, memory_percent, disk_percent_free FROM clients WHERE status = 'active'")
+        client_rows = cursor.fetchall()
+    except Exception:
+        client_rows = []
     conn.close()
 
     window_hours = SERVER_STATE.get("config", {}).get("evaluation_window_hours", 12)
     report = []
+
+    # Map node telemetry and detect silent nodes
+    client_metrics_map = {}
+    for cl in client_rows:
+        cid, chost, cost, clast, cmetrics_raw, ccpu, cmem, cdisk = cl
+        try:
+            m_dict = json.loads(cmetrics_raw) if cmetrics_raw else {}
+        except Exception:
+            m_dict = {}
+        if not m_dict:
+            m_dict = {
+                "cpu_percent": ccpu or 0.0,
+                "memory_percent": cmem or 0.0,
+                "disk_percent_free": cdisk or 0.0
+            }
+        client_metrics_map[cid] = m_dict
+        if chost:
+            client_metrics_map[chost] = m_dict
+
+        # Liveness check: If active client has not checked in for > 15 minutes, report CRITICAL offline anomaly
+        if clast:
+            try:
+                clast_clean = str(clast).replace("T", " ").split("+")[0].split("Z")[0].strip()
+                last_dt = datetime.fromisoformat(clast_clean).replace(tzinfo=timezone.utc)
+                if (now - last_dt) > timedelta(minutes=15):
+                    offline_mins = int((now - last_dt).total_seconds() // 60)
+                    report.append({
+                        "fingerprint": f"offline-node-{cid}",
+                        "site": "default",
+                        "server": chost or cid,
+                        "signature": "LOGAR/NodeSilent",
+                        "severity": "CRITICAL",
+                        "message": f"Forwarder node '{chost or cid}' has gone silent! Last heartbeat was {offline_mins} minutes ago ({clast}). Potential node crash, network partition, or power failure.",
+                        "os_type": cost or "unknown",
+                        "first_seen": clast,
+                        "last_seen": clast,
+                        "consecutive_runs": 1,
+                        "evaluation_window": f"{window_hours}h",
+                        "verified": True,
+                        "status": "NODE_OFFLINE",
+                        "metrics": m_dict
+                    })
+            except Exception:
+                pass
+
     for r in rows:
+        srv_name = r[2]
         report.append({
             "fingerprint": r[0],
             "site": r[1],
-            "server": r[2],
+            "server": srv_name,
             "signature": r[3],
             "severity": r[4],
             "message": decrypt_field_from_storage(r[5]),
@@ -2763,7 +2936,8 @@ def get_verified_anomalies_for_hermes(auth: str = Depends(require_hermes_or_admi
             "consecutive_runs": r[9],
             "evaluation_window": f"{window_hours}h",
             "verified": True,
-            "status": r[10]
+            "status": r[10],
+            "metrics": client_metrics_map.get(srv_name, {})
         })
 
     return report
@@ -3847,6 +4021,7 @@ def serve_webui(request: Request = None):
                 </div>
             </div>
             <div class="header-actions">
+                <button onclick="resetEnrollmentSecret()" class="btn btn-outline" title="Reset shared enrollment secret key and invalidate old keys">🔄 Reset Shared Key</button>
                 <button onclick="openExportProfileModal()" class="btn btn-primary">📥 Export Client Profile</button>
                 <div class="user-menu-container" tabindex="0">
                     <div class="user-tag" onclick="toggleUserDropdown(event)">
@@ -3923,7 +4098,10 @@ def serve_webui(request: Request = None):
             <div class="card">
                 <div class="card-header">
                     <span class="card-title" style="font-size: 15px;">Enrolled Forwarders & Seat Management</span>
-                    <button onclick="openExportProfileModal()" class="btn btn-accent" style="padding:6px 12px; font-size:12px;">+ Export New Client Profile</button>
+                    <div style="display: flex; gap: 8px;">
+                        <button onclick="resetEnrollmentSecret()" class="btn btn-outline" style="padding:6px 12px; font-size:12px;" title="Reset shared enrollment secret key">🔄 Reset Shared Key</button>
+                        <button onclick="openExportProfileModal()" class="btn btn-accent" style="padding:6px 12px; font-size:12px;">+ Export New Client Profile</button>
+                    </div>
                 </div>
                 <div style="overflow-x: auto;">
                     <table>
@@ -4101,7 +4279,10 @@ def serve_webui(request: Request = None):
             </div>
 
             <div style="background: #0f172a; padding: 12px; border-radius: 8px; border: 1px solid var(--border); margin-bottom: 16px;">
-                <div style="font-size: 11px; font-weight: 700; text-transform: uppercase; color: var(--text-muted); margin-bottom: 6px;">Edge Client Quick Deploy Command</div>
+                <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 6px;">
+                    <div style="font-size: 11px; font-weight: 700; text-transform: uppercase; color: var(--text-muted);">Edge Client Quick Deploy Command</div>
+                    <button onclick="resetEnrollmentSecret()" class="btn btn-outline" style="padding: 2px 8px; font-size: 10px;" title="Reset shared enrollment secret key">🔄 Reset Key</button>
+                </div>
                 <pre id="quickDeployCmd" style="font-family: 'JetBrains Mono', monospace; font-size: 11px; color: #38bdf8; word-break: break-all; white-space: pre-wrap;">sudo ./compilation/install_linux_client.sh "http://{detected_host}:{config.get('hermes_port', 8443)}" "{config.get('enrollment_secret')}"</pre>
             </div>
 
@@ -4323,6 +4504,29 @@ def serve_webui(request: Request = None):
             }}
         }}
 
+        async function resetEnrollmentSecret() {{
+            if (!confirm('Are you sure you want to reset the shared client enrollment key? Any un-enrolled forwarders using previous configuration keys will need updated configurations.')) return;
+            try {{
+                const res = await fetch('/api/server/reset-enrollment-secret', {{
+                    method: 'POST',
+                    headers: {{ 'Content-Type': 'application/json' }}
+                }});
+                const data = await res.json();
+                if (res.ok) {{
+                    showAlert('Shared enrollment key reset successfully! Old keys are now invalidated.');
+                    const deployEl = document.getElementById('quickDeployCmd');
+                    if (deployEl && data.enrollment_secret) {{
+                        const host = document.getElementById('exportHost') ? document.getElementById('exportHost').value.trim() : '{detected_host}';
+                        deployEl.textContent = `sudo ./compilation/install_linux_client.sh "http://${{host}}:{config.get('hermes_port', 8443)}" "${{data.enrollment_secret}}"`;
+                    }}
+                }} else {{
+                    alert('Reset failed: ' + (data.detail || 'Unknown error'));
+                }}
+            }} catch (err) {{
+                alert('Reset error: ' + err.message);
+            }}
+        }}
+
         function downloadClientProfile() {{
             const host = document.getElementById('exportHost').value.trim();
             const port = document.getElementById('exportPort').value.trim();

src/client_transport.py

@@ -23,7 +23,8 @@ from log_collector import (
     get_machine_identifier,
     is_path_writable,
     get_default_data_dir,
-    resolve_writable_path
+    resolve_writable_path,
+    collect_system_metrics
 )
 
 CONFIG_FILE_NAME = "client_config.json"
@@ -247,7 +248,8 @@ def send_encrypted_logs_over_socket(
             payload = {
                 "server": actual_machine_id,
                 "timestamp": datetime.now(timezone.utc).isoformat(),
-                "logs": logs
+                "logs": logs,
+                "metrics": collect_system_metrics()
             }
             payload_bytes = json.dumps(payload).encode("utf-8")
             frame = struct.pack(">I", len(payload_bytes)) + payload_bytes
@@ -283,7 +285,8 @@ def send_encrypted_logs_over_socket(
     payload = {
         "server": actual_machine_id,
         "timestamp": datetime.now(timezone.utc).isoformat(),
-        "logs": logs
+        "logs": logs,
+        "metrics": collect_system_metrics()
     }
     payload_json = json.dumps(payload)
     pgp_msg = pgpy.PGPMessage.new(payload_json)

src/log_collector.py

@@ -8,6 +8,8 @@ import os
 import sys
 import json
 import socket
+import shutil
+import ctypes
 import subprocess
 import warnings
 from abc import ABC, abstractmethod
@@ -26,6 +28,141 @@ CONFIG_FILE_NAME = "client_config.json"
 STATE_FILE_NAME = "client_state.json"
 
 
+def collect_system_metrics() -> Dict[str, Any]:
+    """
+    Collects cross-platform host health and resource telemetry without requiring external wheels.
+    Returns:
+        {
+            "cpu_percent": float,
+            "memory_percent": float,
+            "memory_total_mb": int,
+            "memory_free_mb": int,
+            "disk_percent_free": float,
+            "disk_total_gb": float,
+            "disk_free_gb": float
+        }
+    """
+    metrics = {
+        "cpu_percent": 0.0,
+        "memory_percent": 0.0,
+        "memory_total_mb": 0,
+        "memory_free_mb": 0,
+        "disk_percent_free": 0.0,
+        "disk_total_gb": 0.0,
+        "disk_free_gb": 0.0
+    }
+    # 1. Disk metrics
+    try:
+        path = "C:\\" if sys.platform == "win32" else "/"
+        total, used, free = shutil.disk_usage(path)
+        metrics["disk_total_gb"] = round(total / (1024 ** 3), 2)
+        metrics["disk_free_gb"] = round(free / (1024 ** 3), 2)
+        if total > 0:
+            metrics["disk_percent_free"] = round((free / total) * 100, 1)
+    except Exception:
+        pass
+
+    # 2. Memory metrics
+    try:
+        if sys.platform == "win32":
+            class MEMORYSTATUSEX(ctypes.Structure):
+                _fields_ = [
+                    ("dwLength", ctypes.c_ulong),
+                    ("dwMemoryLoad", ctypes.c_ulong),
+                    ("ullTotalPhys", ctypes.c_ulonglong),
+                    ("ullAvailPhys", ctypes.c_ulonglong),
+                    ("ullTotalPageFile", ctypes.c_ulonglong),
+                    ("ullAvailPageFile", ctypes.c_ulonglong),
+                    ("ullTotalVirtual", ctypes.c_ulonglong),
+                    ("ullAvailVirtual", ctypes.c_ulonglong),
+                    ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
+                ]
+            stat = MEMORYSTATUSEX()
+            stat.dwLength = ctypes.sizeof(MEMORYSTATUSEX)
+            if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)):
+                metrics["memory_percent"] = float(stat.dwMemoryLoad)
+                metrics["memory_total_mb"] = int(stat.ullTotalPhys / (1024 * 1024))
+                metrics["memory_free_mb"] = int(stat.ullAvailPhys / (1024 * 1024))
+        else:
+            # Linux: read /proc/meminfo
+            if os.path.exists("/proc/meminfo"):
+                mem_total = 0
+                mem_avail = 0
+                with open("/proc/meminfo", "r", encoding="utf-8") as f:
+                    for line in f:
+                        parts = line.split(":")
+                        if len(parts) == 2:
+                            k, v = parts[0].strip(), parts[1].strip().split()[0]
+                            if k == "MemTotal":
+                                mem_total = int(v)
+                            elif k in ("MemAvailable", "MemFree") and mem_avail == 0:
+                                mem_avail = int(v)
+                if mem_total > 0:
+                    metrics["memory_total_mb"] = mem_total // 1024
+                    metrics["memory_free_mb"] = mem_avail // 1024
+                    metrics["memory_percent"] = round(((mem_total - mem_avail) / mem_total) * 100, 1)
+    except Exception:
+        pass
+
+    # 3. CPU metrics
+    try:
+        if hasattr(os, "getloadavg"):
+            load1, _, _ = os.getloadavg()
+            cpu_count = os.cpu_count() or 1
+            metrics["cpu_percent"] = round(min(100.0, (load1 / cpu_count) * 100), 1)
+        else:
+            try:
+                import psutil
+                metrics["cpu_percent"] = float(psutil.cpu_percent(interval=None))
+            except ImportError:
+                metrics["cpu_percent"] = 0.0
+    except Exception:
+        pass
+
+    return metrics
+
+
+def is_routine_service_noise(signature: str, message: str, severity: str) -> bool:
+    """
+    Identifies high-frequency benign background service start/stop cycles
+    (e.g., sppsvc, edgeupdate, trustedinstaller, wuauserv, backgroundTaskHost)
+    to prevent false-positive anomaly fatigue and consecutive_run metric bloat.
+    """
+    if str(severity).upper() != "INFO":
+        return False
+    sig_low = (signature or "").lower()
+    msg_low = (message or "").lower()
+    routine_services = [
+        "sppsvc",
+        "edgeupdate",
+        "trustedinstaller",
+        "wuauserv",
+        "backgroundtaskhost",
+        "service control manager",
+        "systemd",
+        "cron"
+    ]
+    is_routine_svc = any(s in sig_low or s in msg_low for s in routine_services)
+    if is_routine_svc:
+        lifecycle_phrases = [
+            "entered the running state",
+            "entered the stopped state",
+            "entered the start pending state",
+            "entered the stop pending state",
+            "started successfully",
+            "stopped successfully",
+            "service stopped",
+            "service started",
+            "session opened",
+            "session closed",
+            "clean",
+            "reloaded"
+        ]
+        if any(p in msg_low for p in lifecycle_phrases):
+            return True
+    return False
+
+
 def get_machine_identifier() -> str:
     """
     Returns the fully qualified hostname of the machine sending/collecting logs.

tests/test_server.py

@@ -882,6 +882,110 @@ class TestServerComponent(unittest.TestCase):
         # Ingestion cache mode is direct
         self.assertEqual(Server.SERVER_STATE["config"]["cache_mode"], "direct")
 
+    def test_reset_enrollment_secret_and_invalidation(self):
+        """Tests that rotating enrollment secret invalidates the previous secret and updates DB & config."""
+        init_sec = "initial-secret-123456"
+        Server.init_db(self.test_db, enrollment_secret=init_sec, max_seats=5)
+        Server.SERVER_STATE["config"] = {
+            "db_path": self.test_db,
+            "enrollment_secret": init_sec,
+            "config_file_path": self.test_config
+        }
+        with open(self.test_config, "w", encoding="utf-8") as f:
+            json.dump({"enrollment_secret": init_sec, "db_path": self.test_db}, f)
+
+        # 1. Rotate enrollment secret
+        new_sec = Server.rotate_enrollment_secret()
+        self.assertNotEqual(new_sec, init_sec)
+        self.assertEqual(Server.SERVER_STATE["config"]["enrollment_secret"], new_sec)
+
+        # 2. Check DB
+        conn = sqlite3.connect(self.test_db)
+        c = conn.cursor()
+        c.execute("SELECT enrollment_secret FROM license_config WHERE id = 1")
+        db_sec = c.fetchone()[0]
+        conn.close()
+        self.assertEqual(db_sec, new_sec)
+
+        # 3. Check config file
+        with open(self.test_config, "r", encoding="utf-8") as f:
+            cfg_file = json.load(f)
+        self.assertEqual(cfg_file["enrollment_secret"], new_sec)
+
+        # 4. Check export_client_profile automatically generates a new secret
+        resp = Server.export_client_profile(server_host="10.0.0.5", server_port=9443)
+        exported_profile = json.loads(resp.body.decode("utf-8"))
+        self.assertIn("enrollment_secret", exported_profile)
+        self.assertNotEqual(exported_profile["enrollment_secret"], new_sec)
+        self.assertEqual(exported_profile["enrollment_secret"], Server.SERVER_STATE["config"]["enrollment_secret"])
+
+        # 5. Check reset API endpoint
+        reset_res = Server.reset_server_enrollment_secret()
+        self.assertEqual(reset_res["status"], "success")
+        self.assertIn("enrollment_secret", reset_res)
+        self.assertEqual(reset_res["enrollment_secret"], Server.SERVER_STATE["config"]["enrollment_secret"])
+
+    def test_hermes_offline_node_detection_and_metrics(self):
+        """Tests that Hermes reporting detects silent forwarders (>15m) and includes host telemetry metrics."""
+        Server.init_db(self.test_db)
+        Server.SERVER_STATE["config"] = {
+            "db_path": self.test_db,
+            "evaluation_window_hours": 12
+        }
+        conn = sqlite3.connect(self.test_db)
+        now = datetime.now(timezone.utc)
+        thirty_min_ago = (now - timedelta(minutes=30)).strftime("%Y-%m-%d %H:%M:%S")
+        two_min_ago = (now - timedelta(minutes=2)).strftime("%Y-%m-%d %H:%M:%S")
+
+        metrics_node1 = json.dumps({"cpu_percent": 15.5, "memory_percent": 60.0, "disk_percent_free": 45.0})
+        metrics_node2 = json.dumps({"cpu_percent": 88.0, "memory_percent": 95.0, "disk_percent_free": 4.5})
+
+        conn.execute("""
+            INSERT INTO clients (client_id, hostname, os_type, cert_fingerprint, status, last_seen, metrics_json, cpu_percent, memory_percent, disk_percent_free)
+            VALUES ('node-online', 'srv-online.corp', 'windows', 'FP1', 'active', ?, ?, 15.5, 60.0, 45.0)
+        """, (two_min_ago, metrics_node1))
+
+        conn.execute("""
+            INSERT INTO clients (client_id, hostname, os_type, cert_fingerprint, status, last_seen, metrics_json, cpu_percent, memory_percent, disk_percent_free)
+            VALUES ('node-dead', 'srv-dead.corp', 'linux', 'FP2', 'active', ?, ?, 88.0, 95.0, 4.5)
+        """, (thirty_min_ago, metrics_node2))
+
+        # Also insert a verified operational error for the online node
+        conn.execute("""
+            INSERT INTO active_issues (fingerprint, site_name, server, signature, severity, message, os_type, first_seen, last_seen, run_count, status, last_run_id)
+            VALUES ('default:srv-online.corp:AppCrash', 'default', 'srv-online.corp', 'AppCrash', 'ERROR', 'Segmentation fault', 'windows', ?, ?, 2, 'VERIFIED', 'run-1')
+        """, (two_min_ago, two_min_ago))
+
+        conn.commit()
+        conn.close()
+
+        report = Server.get_verified_anomalies_for_hermes()
+        self.assertGreaterEqual(len(report), 2)
+
+        # Verify offline node alert
+        offline_alert = next((r for r in report if r.get("status") == "NODE_OFFLINE"), None)
+        self.assertIsNotNone(offline_alert)
+        self.assertEqual(offline_alert["server"], "srv-dead.corp")
+        self.assertEqual(offline_alert["severity"], "CRITICAL")
+        self.assertIn("has gone silent", offline_alert["message"])
+        self.assertEqual(offline_alert["metrics"]["disk_percent_free"], 4.5)
+
+        # Verify online error has attached telemetry
+        online_err = next((r for r in report if r.get("signature") == "AppCrash"), None)
+        self.assertIsNotNone(online_err)
+        self.assertEqual(online_err["metrics"]["cpu_percent"], 15.5)
+
+    def test_routine_service_noise_filtering(self):
+        """Tests that benign routine background service start/stop cycles are filtered from anomaly spam."""
+        self.assertTrue(Server.is_routine_service_noise("sppsvc", "The Software Protection service has entered the stopped state.", "INFO"))
+        self.assertTrue(Server.is_routine_service_noise("edgeupdate", "The Microsoft Edge Update service entered the running state.", "INFO"))
+        self.assertTrue(Server.is_routine_service_noise("Service Control Manager", "The Windows Update service entered the stopped state.", "INFO"))
+        self.assertTrue(Server.is_routine_service_noise("systemd", "Service started successfully.", "INFO"))
+
+        # Errors or warnings must NOT be filtered
+        self.assertFalse(Server.is_routine_service_noise("sppsvc", "Software Protection activation failed with error 0xC004F074", "ERROR"))
+        self.assertFalse(Server.is_routine_service_noise("Database", "Connection pool exhausted", "WARNING"))
+
 
 if __name__ == "__main__":
     unittest.main()