Implement edge filtering, state tracking, clean out/ directory, and add Gitea CI workflow

Linux_Client.py

@@ -6,7 +6,8 @@ import struct
 import argparse
 import subprocess
 import warnings
-from datetime import datetime, timezone
+from datetime import datetime, timezone, timedelta
+from typing import Optional, Dict, Any, List
 
 # Suppress cryptography / pgpy deprecation notices
 warnings.filterwarnings("ignore")
@@ -14,6 +15,49 @@ warnings.filterwarnings("ignore")
 import pgpy
 
 CONFIG_FILE_NAME = "client_config.json"
+STATE_FILE_NAME = "client_state.json"
+
+
+def get_state_path(config_path: str, custom_state_path: Optional[str] = None) -> str:
+    if custom_state_path:
+        return custom_state_path
+    config_dir = os.path.dirname(os.path.abspath(config_path))
+    return os.path.join(config_dir, STATE_FILE_NAME)
+
+
+def load_state(state_path: str) -> dict:
+    if os.path.exists(state_path):
+        try:
+            with open(state_path, "r", encoding="utf-8") as f:
+                return json.load(f)
+        except Exception as e:
+            print(f"[!] Warning: Failed to read state file '{state_path}': {e}")
+            return {}
+    return {}
+
+
+def save_state(state_path: str, state: dict):
+    try:
+        temp_path = f"{state_path}.tmp"
+        with open(temp_path, "w", encoding="utf-8") as f:
+            json.dump(state, f, indent=2)
+        os.replace(temp_path, state_path)
+    except Exception as e:
+        print(f"[!] Warning: Could not save client state to '{state_path}': {e}")
+
+
+def commit_state(state: dict, state_path: str):
+    if "new_last_cursor" in state:
+        val = state.pop("new_last_cursor")
+        if val:
+            state["last_cursor"] = val
+    if "new_last_timestamp_us" in state:
+        val = state.pop("new_last_timestamp_us")
+        if val:
+            state["last_timestamp_us"] = val
+    if "new_sent_cursors" in state:
+        state["sent_cursors"] = state.pop("new_sent_cursors")
+    save_state(state_path, state)
 
 
 def load_config(config_path: str = CONFIG_FILE_NAME):
@@ -63,23 +107,56 @@ def get_machine_identifier() -> str:
     return hostname
 
 
-def get_recent_linux_logs(hours: int = 6):
+def get_recent_linux_logs(hours: int = 24, state: Optional[dict] = None) -> list:
     """
-    Collects warnings and errors from systemd journalctl over the lookback window.
-    Edge Thinness: Drops INFO and DEBUG entries at the source.
+    Collects info, warnings, and errors from systemd journalctl over the lookback window.
+    Edge Filtering: Retains INFO, WARNING, and ERROR. Strips DEBUG (priority 7) and skips events older than lookback window (default: 24h).
+    State Tracking: Skips events older than lookback window (default 24h) and events
+    that have already been sent in previous runs.
     """
-    cmd = ["journalctl", "--since", f"{hours} hours ago", "-p", "warning", "--output=json"]
-    try:
-        result = subprocess.run(cmd, capture_output=True, text=True, check=False)
-    except FileNotFoundError:
-        print("[!] journalctl command not found. Ensure this script runs on a systemd-enabled Linux system.")
-        return []
-    except Exception as e:
-        print(f"[!] Error running journalctl: {e}")
-        return []
+    last_cursor = None
+    last_timestamp_us = 0.0
+    sent_cursors = set()
+    if state:
+        last_cursor = state.get("last_cursor")
+        try:
+            last_timestamp_us = float(state.get("last_timestamp_us", 0))
+        except (ValueError, TypeError):
+            last_timestamp_us = 0.0
+        sent_cursors = set(state.get("sent_cursors", []))
+
+    cmd = ["journalctl", "--since", f"{hours} hours ago", "-p", "info", "--output=json"]
+    result = None
+
+    if last_cursor:
+        cmd_with_cursor = ["journalctl", "--since", f"{hours} hours ago", "--after-cursor", str(last_cursor), "-p", "info", "--output=json"]
+        try:
+            res = subprocess.run(cmd_with_cursor, capture_output=True, text=True, check=False)
+            if res.returncode == 0:
+                result = res
+        except FileNotFoundError:
+            print("[!] journalctl command not found. Ensure this script runs on a systemd-enabled Linux system.")
+            return []
+        except Exception:
+            pass
+
+    if result is None:
+        try:
+            result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+        except FileNotFoundError:
+            print("[!] journalctl command not found. Ensure this script runs on a systemd-enabled Linux system.")
+            return []
+        except Exception as e:
+            print(f"[!] Error running journalctl: {e}")
+            return []
 
     logs = []
     machine_id = get_machine_identifier()
+    cutoff_epoch_us = (datetime.now(timezone.utc) - timedelta(hours=hours)).timestamp() * 1_000_000
+
+    newest_cursor = None
+    newest_timestamp_us = last_timestamp_us
+    collected_cursors = []
 
     for line in result.stdout.splitlines():
         line_str = line.strip()
@@ -87,13 +164,48 @@ def get_recent_linux_logs(hours: int = 6):
             continue
         try:
             entry = json.loads(line_str)
-            priority = str(entry.get("PRIORITY", "4"))
-            # Priority 0: Emerg, 1: Alert, 2: Crit, 3: Err, 4: Warning.
-            # Strip anything above 4 (5: Notice, 6: Info, 7: Debug)
-            if int(priority) > 4:
+            entry_cursor = entry.get("__CURSOR")
+            entry_ts_us_raw = entry.get("__REALTIME_TIMESTAMP")
+
+            entry_ts_us = 0.0
+            if entry_ts_us_raw:
+                try:
+                    entry_ts_us = float(entry_ts_us_raw)
+                except (ValueError, TypeError):
+                    pass
+
+            # 1. Skip entries older than lookback window (default: 24h)
+            if entry_ts_us and entry_ts_us < cutoff_epoch_us:
                 continue
 
-            sev = "WARNING" if priority == "4" else "ERROR"
+            # 2. Skip already sent events
+            if entry_cursor and (entry_cursor in sent_cursors or entry_cursor == last_cursor):
+                continue
+            if last_timestamp_us > 0 and entry_ts_us > 0 and entry_ts_us < last_timestamp_us:
+                continue
+
+            # Advance newest tracking for new entries
+            if entry_cursor:
+                newest_cursor = entry_cursor
+                collected_cursors.append(entry_cursor)
+            if entry_ts_us > newest_timestamp_us:
+                newest_timestamp_us = entry_ts_us
+
+            priority = int(entry.get("PRIORITY", "6"))
+            # Priority 0: Emerg, 1: Alert, 2: Crit, 3: Err (-> ERROR)
+            # Priority 4: Warning, 5: Notice (-> WARNING)
+            # Priority 6: Info (-> INFO)
+            # Priority 7: Debug (skip)
+            if priority > 6:
+                continue
+
+            if priority <= 3:
+                sev = "ERROR"
+            elif priority in (4, 5):
+                sev = "WARNING"
+            else:
+                sev = "INFO"
+
             logs.append({
                 "server": machine_id,
                 "os_type": "linux",
@@ -104,13 +216,19 @@ def get_recent_linux_logs(hours: int = 6):
         except (json.JSONDecodeError, ValueError):
             continue
 
+    if state is not None:
+        state["new_last_cursor"] = newest_cursor or last_cursor
+        state["new_last_timestamp_us"] = max(newest_timestamp_us, last_timestamp_us)
+        state["new_sent_cursors"] = (list(sent_cursors) + collected_cursors)[-1000:]
+        state["last_run_timestamp"] = datetime.now(timezone.utc).isoformat()
+
     return logs
 
 
 def send_encrypted_logs_over_socket(config: dict, logs: list):
     """
     Encrypts the payload using the server's OpenPGP public key and streams
-    over an authenticated TCP socket. Zero local state is maintained on the client.
+    over an authenticated TCP socket.
     """
     server_host = config["server_host"]
     server_port = int(config["server_port"])
@@ -128,7 +246,7 @@ def send_encrypted_logs_over_socket(config: dict, logs: list):
 
     machine_id = get_machine_identifier()
 
-    # Prepare zero-state candidate batch
+    # Prepare batch
     payload = {
         "server": machine_id,
         "timestamp": datetime.now(timezone.utc).isoformat(),
@@ -178,9 +296,11 @@ def send_encrypted_logs_over_socket(config: dict, logs: list):
 
 
 def main():
-    parser = argparse.ArgumentParser(description="LOGAR Linux Edge Log Forwarder (Zero State)")
+    parser = argparse.ArgumentParser(description="LOGAR Linux Edge Log Forwarder with State Tracking")
     parser.add_argument("--config", default=CONFIG_FILE_NAME, help="Path to client_config.json")
-    parser.add_argument("--hours", type=int, default=6, help="Lookback window in hours for journalctl logs")
+    parser.add_argument("--hours", type=int, default=24, help="Lookback window in hours for journalctl logs (default: 24)")
+    parser.add_argument("--state-file", default=None, help="Path to state tracking file (default: client_state.json next to config)")
+    parser.add_argument("--no-state", action="store_true", help="Disable state tracking and send all events matching lookback window")
     args = parser.parse_args()
 
     try:
@@ -189,14 +309,31 @@ def main():
         print(f"[!] Configuration error: {e}")
         sys.exit(1)
 
+    state_path = get_state_path(args.config, args.state_file)
+    state = None if args.no_state else load_state(state_path)
+
     machine_id = get_machine_identifier()
     print(f"[*] Edge Forwarder Node: {machine_id}")
-    print(f"[*] Scanning Linux journalctl for candidate anomalies (last {args.hours} hours)...")
-    candidate_logs = get_recent_linux_logs(hours=args.hours)
-    print(f"[*] Found {len(candidate_logs)} candidate anomalies (noise stripped at source).")
+    if state and ("last_cursor" in state or "last_timestamp_us" in state):
+        print(f"[*] State tracking active: resuming after previous cursor/timestamp (state file: {state_path})")
+    elif not args.no_state:
+        print(f"[*] State tracking initialized (state file: {state_path})")
+
+    print(f"[*] Scanning Linux journalctl for unsent entries (last {args.hours} hours)...")
+    candidate_logs = get_recent_linux_logs(hours=args.hours, state=state)
+    print(f"[*] Found {len(candidate_logs)} unsent candidate entries (INFO to ERROR, entries > {args.hours}h and already-sent skipped).")
+
+    if not candidate_logs:
+        print("[*] No new unsent events to transmit.")
+        if state is not None:
+            commit_state(state, state_path)
+        return
 
     try:
-        send_encrypted_logs_over_socket(config, candidate_logs)
+        resp = send_encrypted_logs_over_socket(config, candidate_logs)
+        if state is not None and resp and resp.get("status") == "success":
+            commit_state(state, state_path)
+            print(f"[+] State successfully committed to {state_path}")
     except Exception as e:
         print(f"[!] Failed to stream logs to server: {e}")
         sys.exit(1)