Implement edge filtering, state tracking, clean out/ directory, and add Gitea CI workflow

tests/test_linux_client.py

@@ -0,0 +1,202 @@
+import os
+import sys
+import json
+import struct
+import unittest
+import warnings
+
+warnings.filterwarnings("ignore")
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+import Linux_Client
+import pgpy
+from pgpy.constants import PubKeyAlgorithm, KeyFlags, HashAlgorithm, SymmetricKeyAlgorithm, CompressionAlgorithm
+
+
+class TestLinuxClientComponent(unittest.TestCase):
+    def setUp(self):
+        self.dummy_config = "test_linux_client_config.json"
+        # Generate dummy PGP key for testing
+        key = pgpy.PGPKey.new(PubKeyAlgorithm.RSAEncryptOrSign, 2048)
+        uid = pgpy.PGPUID.new("TestHub")
+        key.add_uid(
+            uid,
+            usage={KeyFlags.EncryptCommunications, KeyFlags.EncryptStorage},
+            hashes=[HashAlgorithm.SHA256],
+            ciphers=[SymmetricKeyAlgorithm.AES256],
+            compression=[CompressionAlgorithm.Uncompressed]
+        )
+        self.server_priv = key
+        self.server_pub = key.pubkey
+        self.fingerprint = str(key.pubkey.fingerprint)
+
+        with open(self.dummy_config, "w", encoding="utf-8") as f:
+            json.dump({
+                "server_host": "127.0.0.1",
+                "server_port": 9443,
+                "server_fingerprint": self.fingerprint,
+                "server_public_key": str(self.server_pub),
+                "auth_token": "secret-test-token"
+            }, f)
+
+    def tearDown(self):
+        if os.path.exists(self.dummy_config):
+            try:
+                os.remove(self.dummy_config)
+            except Exception:
+                pass
+
+    def test_client_config_anonymity(self):
+        config = Linux_Client.load_config(self.dummy_config)
+        self.assertNotIn("server_name", config)
+        self.assertNotIn("name", config)
+        self.assertNotIn("site_name", config)
+        self.assertEqual(config["server_fingerprint"], self.fingerprint)
+
+    def test_get_machine_identifier(self):
+        machine_id = Linux_Client.get_machine_identifier()
+        self.assertIsInstance(machine_id, str)
+        self.assertGreater(len(machine_id), 0)
+        self.assertNotEqual(machine_id, "localhost")
+
+    def test_journalctl_parsing_and_priority_filter(self):
+        sample_journal_lines = [
+            json.dumps({"PRIORITY": "3", "SYSLOG_IDENTIFIER": "sshd", "MESSAGE": "Failed password for root"}),
+            json.dumps({"PRIORITY": "4", "SYSLOG_IDENTIFIER": "systemd", "MESSAGE": "Unit entered failed state"}),
+            json.dumps({"PRIORITY": "6", "SYSLOG_IDENTIFIER": "cron", "MESSAGE": "Informational session opened"}),
+            json.dumps({"PRIORITY": "7", "SYSLOG_IDENTIFIER": "debugd", "MESSAGE": "Verbose debugging log"}),
+        ]
+        logs = []
+        machine_id = Linux_Client.get_machine_identifier()
+        for line_str in sample_journal_lines:
+            entry = json.loads(line_str)
+            priority = int(entry.get("PRIORITY", "6"))
+            # Filter: Retain INFO to ERROR (<= 6), drop DEBUG (> 6)
+            if priority > 6:
+                continue
+
+            if priority <= 3:
+                sev = "ERROR"
+            elif priority in (4, 5):
+                sev = "WARNING"
+            else:
+                sev = "INFO"
+
+            logs.append({
+                "server": machine_id,
+                "os_type": "linux",
+                "signature": entry.get("SYSLOG_IDENTIFIER", "unknown"),
+                "severity": sev,
+                "message": entry.get("MESSAGE", "")
+            })
+
+        # Priority 7 (DEBUG) must be stripped, while 3 (ERROR), 4 (WARNING), 6 (INFO) are retained
+        self.assertEqual(len(logs), 3)
+        self.assertEqual(logs[0]["severity"], "ERROR")
+        self.assertEqual(logs[1]["severity"], "WARNING")
+        self.assertEqual(logs[2]["severity"], "INFO")
+
+    def test_encryption_and_decryption(self):
+        config = Linux_Client.load_config(self.dummy_config)
+        pub_key, _ = pgpy.PGPKey.from_blob(config["server_public_key"])
+        payload = {
+            "server": Linux_Client.get_machine_identifier(),
+            "logs": [{"signature": "kernel", "severity": "ERROR", "message": "Kernel panic - not syncing"}]
+        }
+        msg = pgpy.PGPMessage.new(json.dumps(payload))
+        enc = pub_key.encrypt(msg)
+        self.assertTrue(str(enc).startswith("-----BEGIN PGP MESSAGE-----"))
+
+        dec = self.server_priv.decrypt(enc)
+        restored = json.loads(dec.message)
+        self.assertEqual(restored["logs"][0]["signature"], "kernel")
+
+    def test_state_lifecycle(self):
+        state_path = "test_linux_state.json"
+        try:
+            # 1. Load non-existent returns empty dict
+            state = Linux_Client.load_state(state_path)
+            self.assertEqual(state, {})
+
+            # 2. Stage new cursor and timestamp
+            state["new_last_cursor"] = "s=abc;i=123"
+            state["new_last_timestamp_us"] = 1700000000000000
+            state["new_sent_cursors"] = ["s=abc;i=123"]
+
+            # 3. Commit state moves staged keys to permanent and writes atomically
+            Linux_Client.commit_state(state, state_path)
+            self.assertNotIn("new_last_cursor", state)
+            self.assertEqual(state.get("last_cursor"), "s=abc;i=123")
+            self.assertEqual(state.get("last_timestamp_us"), 1700000000000000)
+            self.assertEqual(state.get("sent_cursors"), ["s=abc;i=123"])
+
+            # 4. Reload from disk
+            reloaded = Linux_Client.load_state(state_path)
+            self.assertEqual(reloaded.get("last_cursor"), "s=abc;i=123")
+            self.assertEqual(reloaded.get("last_timestamp_us"), 1700000000000000)
+        finally:
+            if os.path.exists(state_path):
+                os.remove(state_path)
+
+    def test_duplicate_suppression_and_lookback_logic(self):
+        from datetime import datetime, timezone, timedelta
+        now_us = datetime.now(timezone.utc).timestamp() * 1_000_000
+        cutoff_epoch_us = (datetime.now(timezone.utc) - timedelta(hours=24)).timestamp() * 1_000_000
+
+        mock_entries = [
+            # 1. 26 hours old -> skip (> 24h)
+            {"__CURSOR": "c1", "__REALTIME_TIMESTAMP": str(int(now_us - 26 * 3600 * 1_000_000)), "PRIORITY": "3", "MESSAGE": "Old error"},
+            # 2. 2 hours old, already sent -> skip
+            {"__CURSOR": "c2", "__REALTIME_TIMESTAMP": str(int(now_us - 2 * 3600 * 1_000_000)), "PRIORITY": "4", "MESSAGE": "Already sent warning"},
+            # 3. 1 hour old, new entry -> retain
+            {"__CURSOR": "c3", "__REALTIME_TIMESTAMP": str(int(now_us - 1 * 3600 * 1_000_000)), "PRIORITY": "6", "MESSAGE": "New info"},
+            # 4. 30 mins old, debug -> skip priority
+            {"__CURSOR": "c4", "__REALTIME_TIMESTAMP": str(int(now_us - 1800 * 1_000_000)), "PRIORITY": "7", "MESSAGE": "Debug entry"}
+        ]
+
+        state = {
+            "last_cursor": "c2",
+            "last_timestamp_us": int(now_us - 2 * 3600 * 1_000_000),
+            "sent_cursors": ["c2"]
+        }
+
+        # Simulate the filtering loop from get_recent_linux_logs
+        logs = []
+        last_cursor = state.get("last_cursor")
+        last_timestamp_us = float(state.get("last_timestamp_us", 0))
+        sent_cursors = set(state.get("sent_cursors", []))
+        newest_cursor = None
+        newest_timestamp_us = last_timestamp_us
+        collected_cursors = []
+
+        for entry in mock_entries:
+            entry_cursor = entry.get("__CURSOR")
+            entry_ts_us = float(entry.get("__REALTIME_TIMESTAMP"))
+
+            if entry_ts_us < cutoff_epoch_us:
+                continue
+            if entry_cursor and (entry_cursor in sent_cursors or entry_cursor == last_cursor):
+                continue
+            if last_timestamp_us > 0 and entry_ts_us < last_timestamp_us:
+                continue
+
+            if entry_cursor:
+                newest_cursor = entry_cursor
+                collected_cursors.append(entry_cursor)
+            if entry_ts_us > newest_timestamp_us:
+                newest_timestamp_us = entry_ts_us
+
+            priority = int(entry.get("PRIORITY", "6"))
+            if priority > 6:
+                continue
+
+            logs.append(entry)
+
+        self.assertEqual(len(logs), 1)
+        self.assertEqual(logs[0]["__CURSOR"], "c3")
+        self.assertEqual(newest_cursor, "c4")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_server.py

@@ -0,0 +1,146 @@
+import os
+import sys
+import json
+import socket
+import struct
+import sqlite3
+import unittest
+import urllib.request
+import warnings
+from datetime import datetime, timezone, timedelta
+
+warnings.filterwarnings("ignore")
+
+# Ensure parent directory is in path to import Server
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+import Server
+import pgpy
+
+
+class TestServerComponent(unittest.TestCase):
+    def setUp(self):
+        self.test_db = "test_server_state.db"
+        self.test_config = "test_server_config.json"
+        if os.path.exists(self.test_db):
+            os.remove(self.test_db)
+        if os.path.exists(self.test_config):
+            os.remove(self.test_config)
+
+    def tearDown(self):
+        if os.path.exists(self.test_db):
+            try:
+                os.remove(self.test_db)
+            except Exception:
+                pass
+        if os.path.exists(self.test_config):
+            try:
+                os.remove(self.test_config)
+            except Exception:
+                pass
+
+    def test_first_run_config_and_keypair_generation(self):
+        config = Server.load_or_init_config(self.test_config)
+        self.assertTrue(os.path.exists(self.test_config))
+        self.assertIn("server_fingerprint", config)
+        self.assertIn("public_key", config)
+        self.assertIn("private_key", config)
+        self.assertIn("auth_token", config)
+        self.assertNotIn("site_name", config)
+
+        # Verify keypair
+        priv_key, _ = pgpy.PGPKey.from_blob(config["private_key"])
+        pub_key, _ = pgpy.PGPKey.from_blob(config["public_key"])
+        self.assertEqual(str(pub_key.fingerprint), config["server_fingerprint"])
+
+    def test_create_client_config(self):
+        Server.load_or_init_config(self.test_config)
+        client_out = "test_client_out.json"
+        try:
+            client_conf = Server.create_client_config(
+                server_host="10.0.0.1",
+                server_port=9443,
+                output_path=client_out,
+                config_path=self.test_config
+            )
+            self.assertTrue(os.path.exists(client_out))
+            self.assertEqual(client_conf["server_host"], "10.0.0.1")
+            self.assertEqual(client_conf["server_port"], 9443)
+            # Verify no machine name or site_name is included
+            self.assertNotIn("server_name", client_conf)
+            self.assertNotIn("name", client_conf)
+            self.assertNotIn("site_name", client_conf)
+        finally:
+            if os.path.exists(client_out):
+                os.remove(client_out)
+
+    def test_4_run_rule_and_12h_window(self):
+        Server.init_db(self.test_db)
+        log_entry = {
+            "server": "app-worker-01.corp.local",
+            "signature": "PostgresConnTimeout",
+            "severity": "ERROR",
+            "message": "Connection to database pool timed out after 30s",
+            "os_type": "linux"
+        }
+        payload = {
+            "server": "app-worker-01.corp.local",
+            "logs": [log_entry]
+        }
+
+        # Runs 1 to 3: should remain TRANSIENT
+        for run_idx in range(1, 4):
+            res = Server.process_ingested_logs(payload, self.test_db, window_hours=12, min_runs=4)
+            self.assertEqual(res["status"], "success")
+            self.assertEqual(res["promoted_verified"], 0)
+
+        conn = sqlite3.connect(self.test_db)
+        c = conn.cursor()
+        c.execute("SELECT run_count, status FROM active_issues WHERE signature = ?", ("PostgresConnTimeout",))
+        row = c.fetchone()
+        conn.close()
+        self.assertEqual(row[0], 3)
+        self.assertEqual(row[1], "TRANSIENT")
+
+        # Run 4: promotes to VERIFIED!
+        res4 = Server.process_ingested_logs(payload, self.test_db, window_hours=12, min_runs=4)
+        self.assertEqual(res4["promoted_verified"], 1)
+
+        conn = sqlite3.connect(self.test_db)
+        c = conn.cursor()
+        c.execute("SELECT run_count, status FROM active_issues WHERE signature = ?", ("PostgresConnTimeout",))
+        row = c.fetchone()
+        conn.close()
+        self.assertEqual(row[0], 4)
+        self.assertEqual(row[1], "VERIFIED")
+
+    def test_server_severity_filtering(self):
+        Server.init_db(self.test_db)
+        payload = {
+            "server": "app-worker-01.corp.local",
+            "logs": [
+                {"server": "app-worker-01", "signature": "SigInfo", "severity": "INFO", "message": "Info msg", "os_type": "linux"},
+                {"server": "app-worker-01", "signature": "SigWarn", "severity": "WARNING", "message": "Warn msg", "os_type": "linux"},
+                {"server": "app-worker-01", "signature": "SigErr", "severity": "ERROR", "message": "Err msg", "os_type": "linux"},
+                {"server": "app-worker-01", "signature": "SigDebug", "severity": "DEBUG", "message": "Debug msg", "os_type": "linux"},
+                {"server": "app-worker-01", "signature": "SigTrace", "severity": "TRACE", "message": "Trace msg", "os_type": "linux"}
+            ]
+        }
+        res = Server.process_ingested_logs(payload, self.test_db, window_hours=12, min_runs=4)
+        self.assertEqual(res["status"], "success")
+
+        conn = sqlite3.connect(self.test_db)
+        c = conn.cursor()
+        c.execute("SELECT signature FROM active_issues ORDER BY signature")
+        sigs = [r[0] for r in c.fetchall()]
+        conn.close()
+
+        self.assertIn("SigInfo", sigs)
+        self.assertIn("SigWarn", sigs)
+        self.assertIn("SigErr", sigs)
+        self.assertNotIn("SigDebug", sigs)
+        self.assertNotIn("SigTrace", sigs)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_win_client.py

@@ -0,0 +1,188 @@
+import os
+import sys
+import json
+import struct
+import unittest
+import warnings
+
+warnings.filterwarnings("ignore")
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+import Win_Client
+import pgpy
+from pgpy.constants import PubKeyAlgorithm, KeyFlags, HashAlgorithm, SymmetricKeyAlgorithm, CompressionAlgorithm
+
+
+class TestWinClientComponent(unittest.TestCase):
+    def setUp(self):
+        self.dummy_config = "test_win_client_config.json"
+        # Generate dummy PGP key for testing
+        key = pgpy.PGPKey.new(PubKeyAlgorithm.RSAEncryptOrSign, 2048)
+        uid = pgpy.PGPUID.new("TestHub")
+        key.add_uid(
+            uid,
+            usage={KeyFlags.EncryptCommunications, KeyFlags.EncryptStorage},
+            hashes=[HashAlgorithm.SHA256],
+            ciphers=[SymmetricKeyAlgorithm.AES256],
+            compression=[CompressionAlgorithm.Uncompressed]
+        )
+        self.server_priv = key
+        self.server_pub = key.pubkey
+        self.fingerprint = str(key.pubkey.fingerprint)
+
+        with open(self.dummy_config, "w", encoding="utf-8") as f:
+            json.dump({
+                "server_host": "127.0.0.1",
+                "server_port": 9443,
+                "server_fingerprint": self.fingerprint,
+                "server_public_key": str(self.server_pub),
+                "auth_token": "secret-test-token"
+            }, f)
+
+    def tearDown(self):
+        if os.path.exists(self.dummy_config):
+            try:
+                os.remove(self.dummy_config)
+            except Exception:
+                pass
+
+    def test_client_config_anonymity(self):
+        config = Win_Client.load_config(self.dummy_config)
+        self.assertNotIn("server_name", config)
+        self.assertNotIn("name", config)
+        self.assertNotIn("site_name", config)
+        self.assertEqual(config["server_fingerprint"], self.fingerprint)
+
+    def test_get_machine_identifier(self):
+        machine_id = Win_Client.get_machine_identifier()
+        self.assertIsInstance(machine_id, str)
+        self.assertGreater(len(machine_id), 0)
+        self.assertNotEqual(machine_id, "localhost")
+
+    def test_encryption_and_envelope_creation(self):
+        config = Win_Client.load_config(self.dummy_config)
+        logs = [{
+            "server": Win_Client.get_machine_identifier(),
+            "signature": "TestWinSignature",
+            "severity": "WARNING",
+            "message": "Disk space threshold warning"
+        }]
+
+        pub_key, _ = pgpy.PGPKey.from_blob(config["server_public_key"])
+        payload = {
+            "server": Win_Client.get_machine_identifier(),
+            "logs": logs
+        }
+        msg = pgpy.PGPMessage.new(json.dumps(payload))
+        enc = pub_key.encrypt(msg)
+        self.assertTrue(str(enc).startswith("-----BEGIN PGP MESSAGE-----"))
+
+        # Decrypt with private key to verify end-to-end payload integrity
+        dec = self.server_priv.decrypt(enc)
+        restored = json.loads(dec.message)
+        self.assertEqual(restored["logs"][0]["signature"], "TestWinSignature")
+
+    def test_framing_protocol(self):
+        envelope_data = json.dumps({"test": "data"}).encode("utf-8")
+        frame = struct.pack(">I", len(envelope_data)) + envelope_data
+        self.assertEqual(len(frame), 4 + len(envelope_data))
+        length = struct.unpack(">I", frame[:4])[0]
+        self.assertEqual(length, len(envelope_data))
+
+    def test_windows_event_filtering_and_severity_map(self):
+        # sev_map: 1 -> ERROR, 2 -> WARNING, 4 -> INFO
+        sev_map = {1: "ERROR", 2: "WARNING", 4: "INFO"}
+        raw_event_types = [1, 2, 4, 8, 16]  # 8 is Audit Success, 16 is Audit Failure
+        filtered = [sev_map[et] for et in raw_event_types if et in sev_map]
+        self.assertEqual(filtered, ["ERROR", "WARNING", "INFO"])
+
+    def test_state_lifecycle(self):
+        state_path = "test_win_state.json"
+        try:
+            # 1. Load non-existent returns empty dict
+            state = Win_Client.load_state(state_path)
+            self.assertEqual(state, {})
+
+            # 2. Stage new record number and sent IDs
+            state["new_last_record_number"] = 42
+            state["new_sent_record_ids"] = ["42:2026-09-04T12:00:00"]
+
+            # 3. Commit state moves staged keys to permanent and writes atomically
+            Win_Client.commit_state(state, state_path)
+            self.assertNotIn("new_last_record_number", state)
+            self.assertEqual(state.get("last_record_number"), 42)
+            self.assertEqual(state.get("sent_record_ids"), ["42:2026-09-04T12:00:00"])
+
+            # 4. Reload from disk
+            reloaded = Win_Client.load_state(state_path)
+            self.assertEqual(reloaded.get("last_record_number"), 42)
+            self.assertEqual(reloaded.get("sent_record_ids"), ["42:2026-09-04T12:00:00"])
+        finally:
+            if os.path.exists(state_path):
+                os.remove(state_path)
+
+    def test_duplicate_suppression_and_lookback_logic(self):
+        from datetime import datetime, timezone, timedelta
+        now = datetime.now()
+        cutoff_time = now - timedelta(hours=24)
+
+        # Mock event object
+        class MockEvent:
+            def __init__(self, rec_num, time_gen, event_type, source="TestApp", inserts=None):
+                self.RecordNumber = rec_num
+                self.TimeGenerated = time_gen
+                self.EventType = event_type
+                self.SourceName = source
+                self.StringInserts = inserts or ["Test"]
+
+        # Events read backwards: newest (rec 103) down to older (rec 99)
+        mock_events = [
+            # 1. New error within last 24h
+            MockEvent(103, now - timedelta(hours=1), 1),
+            # 2. New warning within last 24h
+            MockEvent(102, now - timedelta(hours=2), 2),
+            # 3. Already sent event (rec 101)
+            MockEvent(101, now - timedelta(hours=3), 4),
+            # 4. Event at or before last_record_number (rec 100) -> should stop backwards scan
+            MockEvent(100, now - timedelta(hours=4), 1),
+            # 5. Old event (> 24h)
+            MockEvent(99, now - timedelta(hours=26), 1),
+        ]
+
+        state = {
+            "last_record_number": 100,
+            "sent_record_ids": ["101:" + (now - timedelta(hours=3)).isoformat()]
+        }
+
+        sev_map = {1: "ERROR", 2: "WARNING", 4: "INFO"}
+        logs = []
+        last_record_number = int(state.get("last_record_number", 0))
+        sent_record_ids = set(state.get("sent_record_ids", []))
+        newest_record_number = 0
+
+        for event in mock_events:
+            rec_num = int(event.RecordNumber)
+            if newest_record_number == 0:
+                newest_record_number = rec_num
+
+            if event.TimeGenerated < cutoff_time:
+                break
+
+            if last_record_number > 0 and newest_record_number >= last_record_number:
+                if rec_num <= last_record_number:
+                    break
+
+            rec_id = f"{rec_num}:{event.TimeGenerated.isoformat()}"
+            if rec_id in sent_record_ids:
+                continue
+
+            if event.EventType in sev_map:
+                logs.append(rec_num)
+
+        # Only rec 103 and 102 should be processed (101 is already sent, <= 100 breaks early)
+        self.assertEqual(logs, [103, 102])
+
+
+if __name__ == "__main__":
+    unittest.main()