Initial commit: LOGAR edge-thin log analysis system with OpenPGP encryption, authenticated TCP sockets, 4-run persistence rule, Hermes reporting, and modular shippables

out/linux_client/Linux_Client.py

@@ -0,0 +1,194 @@
+import os
+import sys
+import json
+import socket
+import struct
+import argparse
+import subprocess
+import warnings
+from datetime import datetime, timezone
+
+# Suppress cryptography / pgpy deprecation notices
+warnings.filterwarnings("ignore")
+
+import pgpy
+
+CONFIG_FILE_NAME = "client_config.json"
+
+
+def load_config(config_path: str = CONFIG_FILE_NAME):
+    if not os.path.exists(config_path):
+        raise FileNotFoundError(
+            f"Client configuration file not found at: {config_path}\n"
+            f"Generate one from the server using: python Server.py --create-client-config --client-out {config_path}"
+        )
+    with open(config_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def get_machine_identifier() -> str:
+    """
+    Returns the hostname of the machine sending the logs,
+    and appends the network/DNS domain if available.
+    """
+    fqdn = socket.getfqdn()
+    if fqdn and "." in fqdn and not fqdn.startswith("localhost"):
+        return fqdn
+
+    hostname = socket.gethostname()
+
+    try:
+        if os.path.exists("/etc/resolv.conf"):
+            with open("/etc/resolv.conf", "r", encoding="utf-8") as f:
+                for line in f:
+                    parts = line.strip().split()
+                    if parts and parts[0] in ["domain", "search"] and len(parts) > 1:
+                        domain = parts[1]
+                        if domain and not domain.startswith("."):
+                            return f"{hostname}.{domain}"
+    except Exception:
+        pass
+
+    try:
+        host_ip = socket.gethostbyname(hostname)
+        canonical_name = socket.gethostbyaddr(host_ip)[0]
+        if canonical_name and "." in canonical_name and not canonical_name.startswith("localhost"):
+            return canonical_name
+    except Exception:
+        pass
+
+    return hostname
+
+
+def get_recent_linux_logs(hours: int = 6):
+    """
+    Collects warnings and errors from systemd journalctl over the lookback window.
+    Edge Thinness: Drops INFO and DEBUG entries at the source.
+    """
+    cmd = ["journalctl", "--since", f"{hours} hours ago", "-p", "warning", "--output=json"]
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+    except FileNotFoundError:
+        print("[!] journalctl command not found. Ensure this script runs on a systemd-enabled Linux system.")
+        return []
+    except Exception as e:
+        print(f"[!] Error running journalctl: {e}")
+        return []
+
+    logs = []
+    machine_id = get_machine_identifier()
+
+    for line in result.stdout.splitlines():
+        line_str = line.strip()
+        if not line_str:
+            continue
+        try:
+            entry = json.loads(line_str)
+            priority = str(entry.get("PRIORITY", "4"))
+            if int(priority) > 4:
+                continue
+
+            sev = "WARNING" if priority == "4" else "ERROR"
+            logs.append({
+                "server": machine_id,
+                "os_type": "linux",
+                "signature": entry.get("SYSLOG_IDENTIFIER", "unknown"),
+                "severity": sev,
+                "message": entry.get("MESSAGE", "")[:2048]
+            })
+        except (json.JSONDecodeError, ValueError):
+            continue
+
+    return logs
+
+
+def send_encrypted_logs_over_socket(config: dict, logs: list):
+    """
+    Encrypts the payload using the server's OpenPGP public key and streams
+    over an authenticated TCP socket. Zero local state is maintained on the client.
+    """
+    server_host = config["server_host"]
+    server_port = int(config["server_port"])
+    auth_token = config["auth_token"]
+    pub_key_armored = config["server_public_key"]
+    expected_fp = config.get("server_fingerprint", "").replace(" ", "").upper()
+
+    pub_key, _ = pgpy.PGPKey.from_blob(pub_key_armored)
+    actual_fp = str(pub_key.fingerprint).replace(" ", "").upper()
+    if expected_fp and actual_fp != expected_fp:
+        raise ValueError(
+            f"Server fingerprint mismatch! Expected {expected_fp}, but key has {actual_fp}."
+        )
+
+    machine_id = get_machine_identifier()
+
+    payload = {
+        "server": machine_id,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "logs": logs
+    }
+    payload_json = json.dumps(payload)
+
+    pgp_msg = pgpy.PGPMessage.new(payload_json)
+    encrypted_msg = pub_key.encrypt(pgp_msg)
+    encrypted_armored = str(encrypted_msg)
+
+    envelope = {
+        "auth_token": auth_token,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "encrypted_payload": encrypted_armored
+    }
+    envelope_bytes = json.dumps(envelope).encode("utf-8")
+
+    print(f"[*] Connecting to LOGAR server at {server_host}:{server_port} over secure TCP socket...")
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.settimeout(15.0)
+        sock.connect((server_host, server_port))
+
+        frame = struct.pack(">I", len(envelope_bytes)) + envelope_bytes
+        sock.sendall(frame)
+
+        resp_len_bytes = sock.recv(4)
+        if not resp_len_bytes:
+            raise ConnectionError("Server closed connection without response.")
+
+        resp_len = struct.unpack(">I", resp_len_bytes)[0]
+        resp_bytes = bytearray()
+        while len(resp_bytes) < resp_len:
+            chunk = sock.recv(min(4096, resp_len - len(resp_bytes)))
+            if not chunk:
+                break
+            resp_bytes.extend(chunk)
+
+        response = json.loads(resp_bytes.decode("utf-8"))
+        print(f"[+] Server response: {response}")
+        return response
+
+
+def main():
+    parser = argparse.ArgumentParser(description="LOGAR Linux Edge Log Forwarder (Zero State)")
+    parser.add_argument("--config", default=CONFIG_FILE_NAME, help="Path to client_config.json")
+    parser.add_argument("--hours", type=int, default=6, help="Lookback window in hours for journalctl logs")
+    args = parser.parse_args()
+
+    try:
+        config = load_config(args.config)
+    except Exception as e:
+        print(f"[!] Configuration error: {e}")
+        sys.exit(1)
+
+    machine_id = get_machine_identifier()
+    print(f"[*] Edge Forwarder Node: {machine_id}")
+    print(f"[*] Scanning Linux journalctl for candidate anomalies (last {args.hours} hours)...")
+    candidate_logs = get_recent_linux_logs(hours=args.hours)
+    print(f"[*] Found {len(candidate_logs)} candidate anomalies (noise stripped at source).")
+
+    try:
+        send_encrypted_logs_over_socket(config, candidate_logs)
+    except Exception as e:
+        print(f"[!] Failed to stream logs to server: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

out/linux_client/README.md

@@ -0,0 +1,62 @@
+# LOGAR Linux Edge Forwarder
+
+Lightweight edge log forwarder for Linux servers running systemd.
+
+## Features
+- **Zero Local State**: No local SQLite database or state tracking on the edge server.
+- **Edge Noise Stripping**: Strips conversational/informational noise (`INFO`, `DEBUG`) directly at the source via `journalctl -p warning`.
+- **End-to-End OpenPGP Encryption**: Encrypts logs using the server's public key; decrypted exclusively on the cloud hub.
+- **Authenticated TCP Socket**: Direct, low-overhead TCP streaming with token authentication.
+- **No GPG Binary Required**: Pure-Python implementation (`pgpy` + `cryptography`).
+
+## Installation
+```bash
+python3 -m pip install -r requirements.txt
+```
+
+## Configuration
+Place `client_config.json` generated by the server (`Server.py --create-client-config`) in the same directory as `Linux_Client.py`.
+
+## Running the Forwarder
+```bash
+python3 Linux_Client.py --hours 6
+```
+
+## Cron / Systemd Timer Deployment
+### Option A: Cron Job (Every 3 hours)
+```bash
+0 */3 * * * cd /opt/logar && /usr/bin/python3 Linux_Client.py --hours 6 >> /var/log/logar_client.log 2>&1
+```
+
+### Option B: Systemd Service & Timer
+1. Create `/etc/systemd/system/logar-forwarder.service`:
+```ini
+[Unit]
+Description=LOGAR Edge Forwarder
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/opt/logar
+ExecStart=/usr/bin/python3 /opt/logar/Linux_Client.py --hours 6
+```
+
+2. Create `/etc/systemd/system/logar-forwarder.timer`:
+```ini
+[Unit]
+Description=Run LOGAR Edge Forwarder every 3 hours
+
+[Timer]
+OnBootSec=5min
+OnUnitActiveSec=3h
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+```
+
+3. Enable and start:
+```bash
+sudo systemctl daemon-reload
+sudo systemctl enable --now logar-forwarder.timer
+```

out/linux_client/build_bin.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Build script to compile Linux_Client into a standalone native ELF binary on Linux
+set -e
+
+echo "[*] Installing build requirements..."
+pip3 install pyinstaller pgpy cryptography standard-imghdr
+
+echo "[*] Compiling Linux_Client native binary..."
+pyinstaller --onefile --clean --name Linux_Client.bin Linux_Client.py
+
+echo "[+] Compilation successful: dist/Linux_Client.bin"

out/linux_client/client_config.sample.json

@@ -0,0 +1,8 @@
+{
+  "server_host": "192.168.1.100",
+  "server_port": 9443,
+  "server_fingerprint": "PASTE_SERVER_FINGERPRINT_HERE",
+  "server_public_key": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n...\n-----END PGP PUBLIC KEY BLOCK-----\n",
+  "auth_token": "PASTE_AUTH_TOKEN_HERE",
+  "site_name": "Frankfurt-DC"
+}

out/linux_client/requirements.txt

@@ -0,0 +1,3 @@
+pgpy>=0.6.0
+standard-imghdr>=3.13.0; python_version >= "3.13"
+cryptography>=42.0.0

out/linux_client/test/test_linux_client.py

@@ -0,0 +1,107 @@
+import os
+import sys
+import json
+import struct
+import unittest
+import warnings
+
+warnings.filterwarnings("ignore")
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+import Linux_Client
+import pgpy
+from pgpy.constants import PubKeyAlgorithm, KeyFlags, HashAlgorithm, SymmetricKeyAlgorithm, CompressionAlgorithm
+
+
+class TestLinuxClientComponent(unittest.TestCase):
+    def setUp(self):
+        self.dummy_config = "test_linux_client_config.json"
+        # Generate dummy PGP key for testing
+        key = pgpy.PGPKey.new(PubKeyAlgorithm.RSAEncryptOrSign, 2048)
+        uid = pgpy.PGPUID.new("TestHub")
+        key.add_uid(
+            uid,
+            usage={KeyFlags.EncryptCommunications, KeyFlags.EncryptStorage},
+            hashes=[HashAlgorithm.SHA256],
+            ciphers=[SymmetricKeyAlgorithm.AES256],
+            compression=[CompressionAlgorithm.Uncompressed]
+        )
+        self.server_priv = key
+        self.server_pub = key.pubkey
+        self.fingerprint = str(key.pubkey.fingerprint)
+
+        with open(self.dummy_config, "w", encoding="utf-8") as f:
+            json.dump({
+                "server_host": "127.0.0.1",
+                "server_port": 9443,
+                "server_fingerprint": self.fingerprint,
+                "server_public_key": str(self.server_pub),
+                "auth_token": "secret-test-token"
+            }, f)
+
+    def tearDown(self):
+        if os.path.exists(self.dummy_config):
+            try:
+                os.remove(self.dummy_config)
+            except Exception:
+                pass
+
+    def test_client_config_anonymity(self):
+        config = Linux_Client.load_config(self.dummy_config)
+        self.assertNotIn("server_name", config)
+        self.assertNotIn("name", config)
+        self.assertNotIn("site_name", config)
+        self.assertEqual(config["server_fingerprint"], self.fingerprint)
+
+    def test_get_machine_identifier(self):
+        machine_id = Linux_Client.get_machine_identifier()
+        self.assertIsInstance(machine_id, str)
+        self.assertGreater(len(machine_id), 0)
+        self.assertNotEqual(machine_id, "localhost")
+
+    def test_journalctl_parsing_and_priority_filter(self):
+        sample_journal_lines = [
+            json.dumps({"PRIORITY": "3", "SYSLOG_IDENTIFIER": "sshd", "MESSAGE": "Failed password for root"}),
+            json.dumps({"PRIORITY": "4", "SYSLOG_IDENTIFIER": "systemd", "MESSAGE": "Unit entered failed state"}),
+            json.dumps({"PRIORITY": "6", "SYSLOG_IDENTIFIER": "cron", "MESSAGE": "Informational session opened"}),
+        ]
+        logs = []
+        machine_id = Linux_Client.get_machine_identifier()
+        for line_str in sample_journal_lines:
+            entry = json.loads(line_str)
+            priority = str(entry.get("PRIORITY", "4"))
+            if int(priority) > 4:
+                continue
+            sev = "WARNING" if priority == "4" else "ERROR"
+            logs.append({
+                "server": machine_id,
+                "os_type": "linux",
+                "signature": entry.get("SYSLOG_IDENTIFIER", "unknown"),
+                "severity": sev,
+                "message": entry.get("MESSAGE", "")
+            })
+
+        # Priority 6 must be stripped (INFO noise)
+        self.assertEqual(len(logs), 2)
+        self.assertEqual(logs[0]["severity"], "ERROR")
+        self.assertEqual(logs[1]["severity"], "WARNING")
+
+    def test_encryption_and_decryption(self):
+        config = Linux_Client.load_config(self.dummy_config)
+        pub_key, _ = pgpy.PGPKey.from_blob(config["server_public_key"])
+        payload = {
+            "server": Linux_Client.get_machine_identifier(),
+            "logs": [{"signature": "kernel", "severity": "ERROR", "message": "Kernel panic - not syncing"}]
+        }
+        msg = pgpy.PGPMessage.new(json.dumps(payload))
+        enc = pub_key.encrypt(msg)
+        self.assertTrue(str(enc).startswith("-----BEGIN PGP MESSAGE-----"))
+
+        dec = self.server_priv.decrypt(enc)
+        restored = json.loads(dec.message)
+        self.assertEqual(restored["logs"][0]["signature"], "kernel")
+
+
+if __name__ == "__main__":
+    unittest.main()