Initial commit: LOGAR edge-thin log analysis system with OpenPGP encryption, authenticated TCP sockets, 4-run persistence rule, Hermes reporting, and modular shippables
This commit is contained in:
+30
@@ -0,0 +1,30 @@
|
|||||||
|
# Virtual Environment
|
||||||
|
venv/
|
||||||
|
env/
|
||||||
|
ENV/
|
||||||
|
|
||||||
|
# Python cache & build
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*$py.class
|
||||||
|
build/
|
||||||
|
dist/
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Database files & state
|
||||||
|
*.db
|
||||||
|
*.sqlite
|
||||||
|
*.sqlite3
|
||||||
|
|
||||||
|
# Live configuration with generated private keys & tokens (samples are tracked)
|
||||||
|
server_config.json
|
||||||
|
client_config.json
|
||||||
|
test_*.json
|
||||||
|
test_*.db
|
||||||
|
|
||||||
|
# IDE & OS files
|
||||||
|
.vscode/
|
||||||
|
.idea/
|
||||||
|
Thumbs.db
|
||||||
|
Desktop.ini
|
||||||
|
.DS_Store
|
||||||
+206
@@ -0,0 +1,206 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import socket
|
||||||
|
import struct
|
||||||
|
import argparse
|
||||||
|
import subprocess
|
||||||
|
import warnings
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
# Suppress cryptography / pgpy deprecation notices
|
||||||
|
warnings.filterwarnings("ignore")
|
||||||
|
|
||||||
|
import pgpy
|
||||||
|
|
||||||
|
CONFIG_FILE_NAME = "client_config.json"
|
||||||
|
|
||||||
|
|
||||||
|
def load_config(config_path: str = CONFIG_FILE_NAME):
|
||||||
|
if not os.path.exists(config_path):
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"Client configuration file not found at: {config_path}\n"
|
||||||
|
f"Generate one from the server using: python Server.py --create-client-config --client-out {config_path}"
|
||||||
|
)
|
||||||
|
with open(config_path, "r", encoding="utf-8") as f:
|
||||||
|
return json.load(f)
|
||||||
|
|
||||||
|
|
||||||
|
def get_machine_identifier() -> str:
|
||||||
|
"""
|
||||||
|
Returns the hostname of the machine sending the logs,
|
||||||
|
and appends the network/DNS domain if available.
|
||||||
|
"""
|
||||||
|
# 1. Try fully-qualified domain name (FQDN)
|
||||||
|
fqdn = socket.getfqdn()
|
||||||
|
if fqdn and "." in fqdn and not fqdn.startswith("localhost"):
|
||||||
|
return fqdn
|
||||||
|
|
||||||
|
hostname = socket.gethostname()
|
||||||
|
|
||||||
|
# 2. Check /etc/resolv.conf domain or search directive
|
||||||
|
try:
|
||||||
|
if os.path.exists("/etc/resolv.conf"):
|
||||||
|
with open("/etc/resolv.conf", "r", encoding="utf-8") as f:
|
||||||
|
for line in f:
|
||||||
|
parts = line.strip().split()
|
||||||
|
if parts and parts[0] in ["domain", "search"] and len(parts) > 1:
|
||||||
|
domain = parts[1]
|
||||||
|
if domain and not domain.startswith("."):
|
||||||
|
return f"{hostname}.{domain}"
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# 3. Try reverse DNS lookup
|
||||||
|
try:
|
||||||
|
host_ip = socket.gethostbyname(hostname)
|
||||||
|
canonical_name = socket.gethostbyaddr(host_ip)[0]
|
||||||
|
if canonical_name and "." in canonical_name and not canonical_name.startswith("localhost"):
|
||||||
|
return canonical_name
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return hostname
|
||||||
|
|
||||||
|
|
||||||
|
def get_recent_linux_logs(hours: int = 6):
|
||||||
|
"""
|
||||||
|
Collects warnings and errors from systemd journalctl over the lookback window.
|
||||||
|
Edge Thinness: Drops INFO and DEBUG entries at the source.
|
||||||
|
"""
|
||||||
|
cmd = ["journalctl", "--since", f"{hours} hours ago", "-p", "warning", "--output=json"]
|
||||||
|
try:
|
||||||
|
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||||
|
except FileNotFoundError:
|
||||||
|
print("[!] journalctl command not found. Ensure this script runs on a systemd-enabled Linux system.")
|
||||||
|
return []
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[!] Error running journalctl: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
logs = []
|
||||||
|
machine_id = get_machine_identifier()
|
||||||
|
|
||||||
|
for line in result.stdout.splitlines():
|
||||||
|
line_str = line.strip()
|
||||||
|
if not line_str:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
entry = json.loads(line_str)
|
||||||
|
priority = str(entry.get("PRIORITY", "4"))
|
||||||
|
# Priority 0: Emerg, 1: Alert, 2: Crit, 3: Err, 4: Warning.
|
||||||
|
# Strip anything above 4 (5: Notice, 6: Info, 7: Debug)
|
||||||
|
if int(priority) > 4:
|
||||||
|
continue
|
||||||
|
|
||||||
|
sev = "WARNING" if priority == "4" else "ERROR"
|
||||||
|
logs.append({
|
||||||
|
"server": machine_id,
|
||||||
|
"os_type": "linux",
|
||||||
|
"signature": entry.get("SYSLOG_IDENTIFIER", "unknown"),
|
||||||
|
"severity": sev,
|
||||||
|
"message": entry.get("MESSAGE", "")[:2048]
|
||||||
|
})
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
continue
|
||||||
|
|
||||||
|
return logs
|
||||||
|
|
||||||
|
|
||||||
|
def send_encrypted_logs_over_socket(config: dict, logs: list):
|
||||||
|
"""
|
||||||
|
Encrypts the payload using the server's OpenPGP public key and streams
|
||||||
|
over an authenticated TCP socket. Zero local state is maintained on the client.
|
||||||
|
"""
|
||||||
|
server_host = config["server_host"]
|
||||||
|
server_port = int(config["server_port"])
|
||||||
|
auth_token = config["auth_token"]
|
||||||
|
pub_key_armored = config["server_public_key"]
|
||||||
|
expected_fp = config.get("server_fingerprint", "").replace(" ", "").upper()
|
||||||
|
|
||||||
|
# Load and verify server public key
|
||||||
|
pub_key, _ = pgpy.PGPKey.from_blob(pub_key_armored)
|
||||||
|
actual_fp = str(pub_key.fingerprint).replace(" ", "").upper()
|
||||||
|
if expected_fp and actual_fp != expected_fp:
|
||||||
|
raise ValueError(
|
||||||
|
f"Server fingerprint mismatch! Expected {expected_fp}, but key has {actual_fp}."
|
||||||
|
)
|
||||||
|
|
||||||
|
machine_id = get_machine_identifier()
|
||||||
|
|
||||||
|
# Prepare zero-state candidate batch
|
||||||
|
payload = {
|
||||||
|
"server": machine_id,
|
||||||
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"logs": logs
|
||||||
|
}
|
||||||
|
payload_json = json.dumps(payload)
|
||||||
|
|
||||||
|
# Encrypt payload with server's encryption-only key
|
||||||
|
pgp_msg = pgpy.PGPMessage.new(payload_json)
|
||||||
|
encrypted_msg = pub_key.encrypt(pgp_msg)
|
||||||
|
encrypted_armored = str(encrypted_msg)
|
||||||
|
|
||||||
|
# Envelope with socket authentication header
|
||||||
|
envelope = {
|
||||||
|
"auth_token": auth_token,
|
||||||
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"encrypted_payload": encrypted_armored
|
||||||
|
}
|
||||||
|
envelope_bytes = json.dumps(envelope).encode("utf-8")
|
||||||
|
|
||||||
|
# Connect over TCP socket and transmit with 4-byte length prefix framing
|
||||||
|
print(f"[*] Connecting to LOGAR server at {server_host}:{server_port} over secure TCP socket...")
|
||||||
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||||
|
sock.settimeout(15.0)
|
||||||
|
sock.connect((server_host, server_port))
|
||||||
|
|
||||||
|
# Send frame: length (4 bytes big-endian) + envelope
|
||||||
|
frame = struct.pack(">I", len(envelope_bytes)) + envelope_bytes
|
||||||
|
sock.sendall(frame)
|
||||||
|
|
||||||
|
# Receive response length
|
||||||
|
resp_len_bytes = sock.recv(4)
|
||||||
|
if not resp_len_bytes:
|
||||||
|
raise ConnectionError("Server closed connection without response.")
|
||||||
|
|
||||||
|
resp_len = struct.unpack(">I", resp_len_bytes)[0]
|
||||||
|
resp_bytes = bytearray()
|
||||||
|
while len(resp_bytes) < resp_len:
|
||||||
|
chunk = sock.recv(min(4096, resp_len - len(resp_bytes)))
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
resp_bytes.extend(chunk)
|
||||||
|
|
||||||
|
response = json.loads(resp_bytes.decode("utf-8"))
|
||||||
|
print(f"[+] Server response: {response}")
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="LOGAR Linux Edge Log Forwarder (Zero State)")
|
||||||
|
parser.add_argument("--config", default=CONFIG_FILE_NAME, help="Path to client_config.json")
|
||||||
|
parser.add_argument("--hours", type=int, default=6, help="Lookback window in hours for journalctl logs")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
config = load_config(args.config)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[!] Configuration error: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
machine_id = get_machine_identifier()
|
||||||
|
print(f"[*] Edge Forwarder Node: {machine_id}")
|
||||||
|
print(f"[*] Scanning Linux journalctl for candidate anomalies (last {args.hours} hours)...")
|
||||||
|
candidate_logs = get_recent_linux_logs(hours=args.hours)
|
||||||
|
print(f"[*] Found {len(candidate_logs)} candidate anomalies (noise stripped at source).")
|
||||||
|
|
||||||
|
try:
|
||||||
|
send_encrypted_logs_over_socket(config, candidate_logs)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[!] Failed to stream logs to server: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,273 @@
|
|||||||
|
# LOGAR: Edge-Thin Log Analysis & Temporal Verification System
|
||||||
|
|
||||||
|
**LOGAR** is an enterprise log aggregation, verification, and anomaly detection architecture designed for heterogeneous server fleets (Windows & Linux). It combines lightweight zero-state edge forwarders with a centralized cloud hub that applies OpenPGP encryption, authenticated TCP streaming, temporal persistence tracking across 12-hour evaluation windows, and an automated 4-run rule to filter out transient infrastructure blips before reporting verified anomalies to **Hermes**.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Table of Contents
|
||||||
|
1. [Core Philosophy](#core-philosophy)
|
||||||
|
2. [Architecture & Data Flow](#architecture--data-flow)
|
||||||
|
3. [Security & Cryptographic Model](#security--cryptographic-model)
|
||||||
|
4. [Cloud-Side Temporal Persistence & 4-Run Rule](#cloud-side-temporal-persistence--4-run-rule)
|
||||||
|
5. [Agentic Hermes Integration](#agentic-hermes-integration)
|
||||||
|
6. [Dynamic Machine & Domain Identification](#dynamic-machine--domain-identification)
|
||||||
|
7. [Repository & Shippables Structure](#repository--shippables-structure)
|
||||||
|
8. [Getting Started & Installation](#getting-started--installation)
|
||||||
|
9. [Running Tests](#running-tests)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Core Philosophy
|
||||||
|
|
||||||
|
### 1. Edge Thinness & Zero State
|
||||||
|
Site agents running on Windows and Linux act strictly as lightweight forwarders:
|
||||||
|
- **No Local Database**: Clients maintain zero state and no local SQLite or cache files.
|
||||||
|
- **Source-Level Noise Stripping**: Conversational, informational, and debugging log noise (`INFO`, `DEBUG`, audit entries) is dropped directly at the source.
|
||||||
|
- **End-to-End Encryption**: Logs are encrypted using the server's OpenPGP public key before leaving the edge node.
|
||||||
|
- **Secure TCP Sockets**: Ingestion occurs over low-overhead authenticated TCP sockets rather than bulky HTTP/HTTPS endpoints.
|
||||||
|
|
||||||
|
### 2. Cloud-Side Temporal Persistence
|
||||||
|
The central Python/TCP hub handles the heavy lifting:
|
||||||
|
- State tracking is managed centrally in SQLite (`logar_state.db`).
|
||||||
|
- Candidate issues are evaluated over a **12-hour temporal evaluation window**.
|
||||||
|
- An issue must persist across **at least 4 consecutive runs / cycles** to be confirmed as a genuine system anomaly. Transient blips and sporadic spikes are filtered out automatically.
|
||||||
|
|
||||||
|
### 3. Agentic Integration with Hermes
|
||||||
|
Instead of human engineers manually diving through noisy logs, **Hermes** ingests pre-filtered, 4-run validated anomalies directly from the cloud hub (`GET /api/hermes/report`), treating them as verified system artifacts to trigger precise team notifications.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Architecture & Data Flow
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph TB
|
||||||
|
subgraph Edge Nodes [Zero-State Edge Forwarders]
|
||||||
|
W[Win_Client.py / Win_Client.exe<br/>Windows Event Log Application]
|
||||||
|
L[Linux_Client.py / Linux_Client.bin<br/>systemd journalctl -p warning]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph Security Layer [Security & Framing]
|
||||||
|
E[OpenPGP Payload Encryption<br/>Server Public Key & Fingerprint]
|
||||||
|
S[Length-Prefixed Framing<br/>4-byte Big-Endian + Auth Envelope]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph Cloud Hub [LOGAR Central Server Hub]
|
||||||
|
TCP[Authenticated TCP Listener<br/>Port 9443]
|
||||||
|
DEC[OpenPGP Decryption<br/>Server Private Key]
|
||||||
|
DB[(SQLite Persistence<br/>active_issues & ingest_runs)]
|
||||||
|
RULE{12h Window &<br/>4-Run Rule}
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph Agentic Reporting [Downstream Integration]
|
||||||
|
API[FastAPI / Uvicorn Reporting<br/>Port 8443]
|
||||||
|
HERMES[Hermes Agent<br/>GET /api/hermes/report]
|
||||||
|
end
|
||||||
|
|
||||||
|
W --> E
|
||||||
|
L --> E
|
||||||
|
E --> S
|
||||||
|
S -->|TCP Stream| TCP
|
||||||
|
TCP --> DEC
|
||||||
|
DEC --> RULE
|
||||||
|
RULE --> DB
|
||||||
|
DB --> API
|
||||||
|
API --> HERMES
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Security & Cryptographic Model
|
||||||
|
|
||||||
|
### Pure-Python OpenPGP (RFC 4880)
|
||||||
|
- **Zero OS Binary Dependency**: Utilizes `pgpy` and `cryptography` in pure Python. **No native GnuPG or `gpg` binary installation is required** on the server, Windows nodes, or Linux nodes.
|
||||||
|
- **First-Run Automatic Key Generation**: On the first launch, if `server_config.json` is missing, `Server.py` automatically generates:
|
||||||
|
- An OpenPGP RSA 2048 keypair with encryption-only usage flags.
|
||||||
|
- An armored private key (`private_key`) and public key (`public_key`).
|
||||||
|
- A SHA-256 public encryption fingerprint (`server_fingerprint`).
|
||||||
|
- A cryptographically random authentication secret token (`auth_token`).
|
||||||
|
- **Client Configuration Exporter**:
|
||||||
|
```bash
|
||||||
|
python Server.py --create-client-config --server-host 127.0.0.1 --server-port 9443 --client-out client_config.json
|
||||||
|
```
|
||||||
|
Produces an anonymous client config containing only the server socket coordinates, authentication token, and the encryption-only public key & fingerprint.
|
||||||
|
- **Socket Protocol Framing**:
|
||||||
|
- `[4 bytes big-endian unsigned int]` : Total envelope length.
|
||||||
|
- `[JSON Envelope]` :
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"auth_token": "<SECRET_TOKEN>",
|
||||||
|
"timestamp": "2026-09-03T...",
|
||||||
|
"encrypted_payload": "-----BEGIN PGP MESSAGE-----\n..."
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- Unauthorized clients or invalid authentication tokens are rejected immediately.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Cloud-Side Temporal Persistence & 4-Run Rule
|
||||||
|
|
||||||
|
Incoming candidate logs are tracked in SQLite table `active_issues`:
|
||||||
|
- **Issue Fingerprint**: Formatted as `{site_name}:{server}:{signature}`.
|
||||||
|
- **12-Hour Evaluation Window**:
|
||||||
|
- When an issue is observed, the hub compares `(now - last_seen)`.
|
||||||
|
- If more than 12 hours have passed since the issue was last recorded, the previous window is expired and the cycle resets to `run_count = 1` with status `TRANSIENT`.
|
||||||
|
- **4-Run Rule**:
|
||||||
|
- For each distinct run batch, `run_count` increments.
|
||||||
|
- Issues with `run_count < 4` are marked as `TRANSIENT` and ignored by downstream reporting.
|
||||||
|
- When `run_count >= 4` within the active 12-hour window, the status transitions to `VERIFIED`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Agentic Hermes Integration
|
||||||
|
|
||||||
|
The server hub serves a REST reporting API (default port `8443`):
|
||||||
|
|
||||||
|
### `GET /api/hermes/report`
|
||||||
|
Returns exclusively **verified anomalies** that have satisfied the 4-run rule within the active 12-hour evaluation window:
|
||||||
|
|
||||||
|
```json
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"fingerprint": "corp.internal:web-app-01.corp.internal:NginxWorkerCrash",
|
||||||
|
"site": "corp.internal",
|
||||||
|
"server": "web-app-01.corp.internal",
|
||||||
|
"signature": "NginxWorkerCrash",
|
||||||
|
"severity": "ERROR",
|
||||||
|
"message": "Worker process 4120 terminated with signal 11",
|
||||||
|
"os_type": "linux",
|
||||||
|
"first_seen": "2026-09-03T09:00:00+00:00",
|
||||||
|
"last_seen": "2026-09-03T21:00:00+00:00",
|
||||||
|
"consecutive_runs": 4,
|
||||||
|
"evaluation_window": "12h",
|
||||||
|
"verified": true,
|
||||||
|
"status": "VERIFIED"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
### `GET /api/hermes/all`
|
||||||
|
Diagnostic endpoint listing all active issues (both `TRANSIENT` candidate blips and `VERIFIED` anomalies).
|
||||||
|
|
||||||
|
### `GET /health`
|
||||||
|
Returns hub health, encryption fingerprint, and listener ports.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Dynamic Machine & Domain Identification
|
||||||
|
|
||||||
|
Client configurations intentionally contain **no machine name or site name**. Both forwarders dynamically identify their host and domain at runtime via `get_machine_identifier()`:
|
||||||
|
1. **Fully Qualified Domain Name (FQDN)**: Checked via `socket.getfqdn()`.
|
||||||
|
2. **OS-Specific Domain Discovery**:
|
||||||
|
- **Windows**: Checks Active Directory environment variable `USERDNSDOMAIN` / `USERDOMAIN`.
|
||||||
|
- **Linux**: Parses `/etc/resolv.conf` `domain` and `search` directives.
|
||||||
|
3. **Reverse DNS Lookup**: Resolves canonical hostname via `socket.gethostbyaddr`.
|
||||||
|
4. **Fallback**: Local hostname `socket.gethostname()`.
|
||||||
|
|
||||||
|
The server automatically infers site attribution from domain qualifiers (e.g. `node01.corp.internal` $\rightarrow$ site `corp.internal`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Repository & Shippables Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
LOGAR/
|
||||||
|
├── .gitignore # Ignore venv, caches, DBs, and private keys
|
||||||
|
├── requirements.txt # Unified dependencies
|
||||||
|
├── README.md # Comprehensive documentation
|
||||||
|
├── Server.py # Central TCP server and Hermes API
|
||||||
|
├── Win_Client.py # Windows edge forwarder
|
||||||
|
├── Linux_Client.py # Linux edge forwarder
|
||||||
|
├── test_pipeline.py # End-to-end integration test
|
||||||
|
└── out/ # Standalone shippable distributions
|
||||||
|
├── server/
|
||||||
|
│ ├── Server.exe # Standalone Windows executable
|
||||||
|
│ ├── Server.py # Python source
|
||||||
|
│ ├── server_config.sample.json
|
||||||
|
│ ├── requirements.txt
|
||||||
|
│ ├── README.md
|
||||||
|
│ └── test/
|
||||||
|
│ └── test_server.py # Server unit tests
|
||||||
|
├── win_client/
|
||||||
|
│ ├── Win_Client.exe # Standalone Windows executable
|
||||||
|
│ ├── Win_Client.py # Python source
|
||||||
|
│ ├── client_config.sample.json
|
||||||
|
│ ├── requirements.txt
|
||||||
|
│ ├── README.md
|
||||||
|
│ └── test/
|
||||||
|
│ └── test_win_client.py # Windows client unit tests
|
||||||
|
└── linux_client/
|
||||||
|
├── Linux_Client.bin # Standalone executable binary (zipapp)
|
||||||
|
├── build_bin.sh # PyInstaller ELF compiler script
|
||||||
|
├── Linux_Client.py # Python source
|
||||||
|
├── client_config.sample.json
|
||||||
|
├── requirements.txt
|
||||||
|
├── README.md
|
||||||
|
└── test/
|
||||||
|
└── test_linux_client.py# Linux client unit tests
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Getting Started & Installation
|
||||||
|
|
||||||
|
### 1. Central Server Hub
|
||||||
|
|
||||||
|
1. **Install dependencies**:
|
||||||
|
```bash
|
||||||
|
pip install -r requirements.txt
|
||||||
|
```
|
||||||
|
2. **Start the server** (generates `server_config.json` and keypair on first run):
|
||||||
|
```bash
|
||||||
|
python Server.py
|
||||||
|
# Or run the standalone executable:
|
||||||
|
./out/server/Server.exe
|
||||||
|
```
|
||||||
|
3. **Export a client configuration**:
|
||||||
|
```bash
|
||||||
|
python Server.py --create-client-config --server-host <SERVER_IP> --server-port 9443 --client-out client_config.json
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Windows Client Deployment
|
||||||
|
|
||||||
|
1. Copy `Win_Client.exe` (or `Win_Client.py`) and `client_config.json` to the target machine.
|
||||||
|
2. Run manually or schedule via Task Scheduler (every 3 hours):
|
||||||
|
```powershell
|
||||||
|
Win_Client.exe --hours 6
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Linux Client Deployment
|
||||||
|
|
||||||
|
1. Copy `Linux_Client.bin` (or `Linux_Client.py`) and `client_config.json` to `/opt/logar/`.
|
||||||
|
2. Ensure executable permissions:
|
||||||
|
```bash
|
||||||
|
chmod +x /opt/logar/Linux_Client.bin
|
||||||
|
```
|
||||||
|
3. Run via cron or systemd timer:
|
||||||
|
```bash
|
||||||
|
0 */3 * * * /opt/logar/Linux_Client.bin --hours 6
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Running Tests
|
||||||
|
|
||||||
|
### 1. Component-Specific Unit Tests
|
||||||
|
Each component in `out/` includes its own isolated test suite:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Server tests (config generation, SQLite persistence, 4-run rule)
|
||||||
|
python out/server/test/test_server.py
|
||||||
|
|
||||||
|
# Windows client tests (config anonymity, machine ID, OpenPGP encryption)
|
||||||
|
python out/win_client/test/test_win_client.py
|
||||||
|
|
||||||
|
# Linux client tests (config anonymity, journalctl priority filter, OpenPGP)
|
||||||
|
python out/linux_client/test/test_linux_client.py
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. End-to-End Pipeline Integration Test
|
||||||
|
Start the server in one shell and run the pipeline test:
|
||||||
|
```bash
|
||||||
|
python test_pipeline.py
|
||||||
|
```
|
||||||
|
This tests invalid token rejection, encrypted socket streaming, database persistence, status promotion upon the 4th run, and the Hermes API output.
|
||||||
@@ -0,0 +1,465 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import uuid
|
||||||
|
import struct
|
||||||
|
import socket
|
||||||
|
import sqlite3
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import secrets
|
||||||
|
import warnings
|
||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
from typing import Dict, Any, List, Optional
|
||||||
|
|
||||||
|
# Suppress cryptography / pgpy deprecation notices for a clean terminal output
|
||||||
|
warnings.filterwarnings("ignore")
|
||||||
|
|
||||||
|
import pgpy
|
||||||
|
from pgpy.constants import (
|
||||||
|
PubKeyAlgorithm,
|
||||||
|
KeyFlags,
|
||||||
|
HashAlgorithm,
|
||||||
|
SymmetricKeyAlgorithm,
|
||||||
|
CompressionAlgorithm
|
||||||
|
)
|
||||||
|
from fastapi import FastAPI, HTTPException
|
||||||
|
import uvicorn
|
||||||
|
|
||||||
|
CONFIG_FILE_NAME = "server_config.json"
|
||||||
|
DEFAULT_DB_FILE = "logar_state.db"
|
||||||
|
EVALUATION_WINDOW_HOURS = 12
|
||||||
|
RUN_THRESHOLD = 4
|
||||||
|
|
||||||
|
app = FastAPI(title="LOGAR Cloud Ingestion & Hermes Hub", version="2.0.0")
|
||||||
|
|
||||||
|
# Global context holding server state
|
||||||
|
SERVER_STATE: Dict[str, Any] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def generate_server_keypair(server_name: str):
|
||||||
|
"""Generates an OpenPGP RSA 2048 key with encryption capability."""
|
||||||
|
key = pgpy.PGPKey.new(PubKeyAlgorithm.RSAEncryptOrSign, 2048)
|
||||||
|
uid = pgpy.PGPUID.new(server_name)
|
||||||
|
key.add_uid(
|
||||||
|
uid,
|
||||||
|
usage={KeyFlags.EncryptCommunications, KeyFlags.EncryptStorage},
|
||||||
|
hashes=[HashAlgorithm.SHA256],
|
||||||
|
ciphers=[SymmetricKeyAlgorithm.AES256],
|
||||||
|
compression=[CompressionAlgorithm.Uncompressed]
|
||||||
|
)
|
||||||
|
private_key_armored = str(key)
|
||||||
|
public_key_armored = str(key.pubkey)
|
||||||
|
fingerprint = str(key.pubkey.fingerprint)
|
||||||
|
return private_key_armored, public_key_armored, fingerprint
|
||||||
|
|
||||||
|
|
||||||
|
def load_or_init_config(config_path: str = CONFIG_FILE_NAME) -> Dict[str, Any]:
|
||||||
|
"""Loads existing server_config.json or creates a new one on first run."""
|
||||||
|
if os.path.exists(config_path):
|
||||||
|
print(f"[*] Loading server configuration from: {os.path.abspath(config_path)}")
|
||||||
|
with open(config_path, "r", encoding="utf-8") as f:
|
||||||
|
config = json.load(f)
|
||||||
|
return config
|
||||||
|
|
||||||
|
print(f"[!] Config '{config_path}' not found. Initializing first-run configuration...")
|
||||||
|
server_name = "LOGAR-Cloud-Hub"
|
||||||
|
private_key, public_key, fingerprint = generate_server_keypair(server_name)
|
||||||
|
auth_token = secrets.token_hex(24)
|
||||||
|
|
||||||
|
config = {
|
||||||
|
"server_name": server_name,
|
||||||
|
"tcp_host": "0.0.0.0",
|
||||||
|
"tcp_port": 9443,
|
||||||
|
"hermes_host": "0.0.0.0",
|
||||||
|
"hermes_port": 8443,
|
||||||
|
"auth_token": auth_token,
|
||||||
|
"db_path": DEFAULT_DB_FILE,
|
||||||
|
"evaluation_window_hours": EVALUATION_WINDOW_HOURS,
|
||||||
|
"min_persistence_runs": RUN_THRESHOLD,
|
||||||
|
"server_fingerprint": fingerprint,
|
||||||
|
"public_key": public_key,
|
||||||
|
"private_key": private_key
|
||||||
|
}
|
||||||
|
|
||||||
|
with open(config_path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(config, f, indent=2)
|
||||||
|
|
||||||
|
print(f"[+] Successfully generated new server config and OpenPGP keypair.")
|
||||||
|
print(f"[+] Server Encryption Fingerprint: {fingerprint}")
|
||||||
|
print(f"[+] Saved to: {os.path.abspath(config_path)}")
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
def create_client_config(
|
||||||
|
server_host: str,
|
||||||
|
server_port: int,
|
||||||
|
output_path: str,
|
||||||
|
config_path: str = CONFIG_FILE_NAME
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Creates a client configuration file containing the server address, auth token, and encryption-only key/fingerprint."""
|
||||||
|
server_conf = load_or_init_config(config_path)
|
||||||
|
|
||||||
|
client_conf = {
|
||||||
|
"server_host": server_host,
|
||||||
|
"server_port": server_port,
|
||||||
|
"server_fingerprint": server_conf["server_fingerprint"],
|
||||||
|
"server_public_key": server_conf["public_key"],
|
||||||
|
"auth_token": server_conf["auth_token"]
|
||||||
|
}
|
||||||
|
|
||||||
|
out_dir = os.path.dirname(os.path.abspath(output_path))
|
||||||
|
if out_dir and not os.path.exists(out_dir):
|
||||||
|
os.makedirs(out_dir, exist_ok=True)
|
||||||
|
|
||||||
|
with open(output_path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(client_conf, f, indent=2)
|
||||||
|
|
||||||
|
print(f"[+] Client configuration successfully written to: {os.path.abspath(output_path)}")
|
||||||
|
print(f" - Server Target: {server_host}:{server_port}")
|
||||||
|
print(f" - Encryption Fingerprint: {server_conf['server_fingerprint']}")
|
||||||
|
return client_conf
|
||||||
|
|
||||||
|
|
||||||
|
def init_db(db_path: str):
|
||||||
|
"""Initializes the SQLite schema for multi-run temporal tracking."""
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS active_issues (
|
||||||
|
fingerprint TEXT PRIMARY KEY,
|
||||||
|
site_name TEXT,
|
||||||
|
server TEXT,
|
||||||
|
signature TEXT,
|
||||||
|
severity TEXT,
|
||||||
|
message TEXT,
|
||||||
|
os_type TEXT,
|
||||||
|
first_seen TEXT,
|
||||||
|
last_seen TEXT,
|
||||||
|
run_count INTEGER,
|
||||||
|
status TEXT,
|
||||||
|
last_run_id TEXT
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS ingest_runs (
|
||||||
|
run_id TEXT PRIMARY KEY,
|
||||||
|
site_name TEXT,
|
||||||
|
server TEXT,
|
||||||
|
timestamp TEXT,
|
||||||
|
log_count INTEGER
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: int, min_runs: int) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Evaluates candidate issues against the 12-hour evaluation window and 4-run rule.
|
||||||
|
Zero-state clients send raw candidate entries; this engine handles temporal state.
|
||||||
|
"""
|
||||||
|
client_server = payload.get("server", "unknown-host")
|
||||||
|
site_name = payload.get("site_name") or (client_server.split(".", 1)[1] if "." in client_server else "default")
|
||||||
|
logs = payload.get("logs", [])
|
||||||
|
run_id = str(uuid.uuid4())
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
now_iso = now.isoformat()
|
||||||
|
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
cursor = conn.cursor()
|
||||||
|
|
||||||
|
# Record the batch run
|
||||||
|
cursor.execute(
|
||||||
|
"INSERT INTO ingest_runs (run_id, site_name, server, timestamp, log_count) VALUES (?, ?, ?, ?, ?)",
|
||||||
|
(run_id, site_name, client_server, now_iso, len(logs))
|
||||||
|
)
|
||||||
|
|
||||||
|
processed_count = 0
|
||||||
|
promoted_to_verified = 0
|
||||||
|
|
||||||
|
for log in logs:
|
||||||
|
severity = str(log.get("severity", "WARNING")).upper()
|
||||||
|
# Edge forwarder filter safeguard (strip informational noise)
|
||||||
|
if severity in ["INFO", "DEBUG"]:
|
||||||
|
continue
|
||||||
|
|
||||||
|
signature = log.get("signature", "unknown")
|
||||||
|
server = log.get("server", client_server)
|
||||||
|
message = log.get("message", "")
|
||||||
|
os_type = log.get("os_type", "unknown")
|
||||||
|
fp = f"{site_name}:{server}:{signature}"
|
||||||
|
|
||||||
|
cursor.execute(
|
||||||
|
"SELECT run_count, first_seen, last_seen, status, last_run_id FROM active_issues WHERE fingerprint = ?",
|
||||||
|
(fp,)
|
||||||
|
)
|
||||||
|
row = cursor.fetchone()
|
||||||
|
|
||||||
|
if row:
|
||||||
|
run_count, first_seen_str, last_seen_str, current_status, last_run_id = row
|
||||||
|
try:
|
||||||
|
last_seen_dt = datetime.fromisoformat(last_seen_str)
|
||||||
|
except Exception:
|
||||||
|
last_seen_dt = now
|
||||||
|
|
||||||
|
# 12-hour evaluation window expiry check
|
||||||
|
if (now - last_seen_dt) > timedelta(hours=window_hours):
|
||||||
|
# Window elapsed: reset to new cycle
|
||||||
|
new_runs = 1
|
||||||
|
new_first_seen = now_iso
|
||||||
|
new_status = "TRANSIENT"
|
||||||
|
else:
|
||||||
|
# Same run guard: only increment count once per distinct run batch
|
||||||
|
if last_run_id != run_id:
|
||||||
|
new_runs = run_count + 1
|
||||||
|
else:
|
||||||
|
new_runs = run_count
|
||||||
|
new_first_seen = first_seen_str
|
||||||
|
# 4-run rule enforcement
|
||||||
|
new_status = "VERIFIED" if new_runs >= min_runs else "TRANSIENT"
|
||||||
|
|
||||||
|
if new_status == "VERIFIED" and current_status != "VERIFIED":
|
||||||
|
promoted_to_verified += 1
|
||||||
|
|
||||||
|
cursor.execute("""
|
||||||
|
UPDATE active_issues
|
||||||
|
SET run_count = ?, last_seen = ?, first_seen = ?, status = ?, last_run_id = ?, message = ?, severity = ?
|
||||||
|
WHERE fingerprint = ?
|
||||||
|
""", (new_runs, now_iso, new_first_seen, new_status, run_id, message, severity, fp))
|
||||||
|
else:
|
||||||
|
initial_status = "VERIFIED" if 1 >= min_runs else "TRANSIENT"
|
||||||
|
cursor.execute("""
|
||||||
|
INSERT INTO active_issues
|
||||||
|
(fingerprint, site_name, server, signature, severity, message, os_type, first_seen, last_seen, run_count, status, last_run_id)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""", (fp, site_name, server, signature, severity, message, os_type, now_iso, now_iso, 1, initial_status, run_id))
|
||||||
|
|
||||||
|
processed_count += 1
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
return {
|
||||||
|
"status": "success",
|
||||||
|
"run_id": run_id,
|
||||||
|
"processed": processed_count,
|
||||||
|
"promoted_verified": promoted_to_verified
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def handle_socket_client(reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
|
||||||
|
"""
|
||||||
|
Authenticated TCP socket handler.
|
||||||
|
Protocol:
|
||||||
|
- 4-byte big-endian prefix: payload length
|
||||||
|
- Payload: JSON with auth_token and encrypted_payload (OpenPGP ASCII armored)
|
||||||
|
- Response: 4-byte length + JSON confirmation
|
||||||
|
"""
|
||||||
|
addr = writer.get_extra_info("peername")
|
||||||
|
try:
|
||||||
|
# Read 4-byte length prefix
|
||||||
|
length_bytes = await reader.readexactly(4)
|
||||||
|
length = struct.unpack(">I", length_bytes)[0]
|
||||||
|
if length <= 0 or length > 10 * 1024 * 1024: # 10MB limit
|
||||||
|
raise ValueError(f"Invalid frame size: {length}")
|
||||||
|
|
||||||
|
payload_bytes = await reader.readexactly(length)
|
||||||
|
envelope = json.loads(payload_bytes.decode("utf-8"))
|
||||||
|
|
||||||
|
# Authenticate socket client
|
||||||
|
expected_token = SERVER_STATE["config"]["auth_token"]
|
||||||
|
provided_token = envelope.get("auth_token")
|
||||||
|
if not secrets.compare_digest(str(provided_token), str(expected_token)):
|
||||||
|
err_msg = json.dumps({"status": "error", "message": "Authentication failed"}).encode("utf-8")
|
||||||
|
writer.write(struct.pack(">I", len(err_msg)) + err_msg)
|
||||||
|
await writer.drain()
|
||||||
|
writer.close()
|
||||||
|
await writer.wait_closed()
|
||||||
|
return
|
||||||
|
|
||||||
|
# Decrypt payload using server's OpenPGP private key
|
||||||
|
encrypted_armored = envelope.get("encrypted_payload", "")
|
||||||
|
pgp_msg = pgpy.PGPMessage.from_blob(encrypted_armored)
|
||||||
|
priv_key = SERVER_STATE["private_key_obj"]
|
||||||
|
decrypted_obj = priv_key.decrypt(pgp_msg)
|
||||||
|
decrypted_json_str = decrypted_obj.message
|
||||||
|
log_payload = json.loads(decrypted_json_str)
|
||||||
|
|
||||||
|
# Ingest and apply 12h window / 4-run rule
|
||||||
|
res = process_ingested_logs(
|
||||||
|
log_payload,
|
||||||
|
db_path=SERVER_STATE["config"]["db_path"],
|
||||||
|
window_hours=SERVER_STATE["config"]["evaluation_window_hours"],
|
||||||
|
min_runs=SERVER_STATE["config"]["min_persistence_runs"]
|
||||||
|
)
|
||||||
|
|
||||||
|
resp_bytes = json.dumps(res).encode("utf-8")
|
||||||
|
writer.write(struct.pack(">I", len(resp_bytes)) + resp_bytes)
|
||||||
|
await writer.drain()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
err = json.dumps({"status": "error", "message": str(e)}).encode("utf-8")
|
||||||
|
try:
|
||||||
|
writer.write(struct.pack(">I", len(err)) + err)
|
||||||
|
await writer.drain()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
writer.close()
|
||||||
|
try:
|
||||||
|
await writer.wait_closed()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/hermes/report")
|
||||||
|
def get_hermes_report():
|
||||||
|
"""
|
||||||
|
Agentic Integration endpoint: Consumed by Hermes to fetch anomalies that have persisted
|
||||||
|
across the 12-hour evaluation window and satisfied the 4-run rule.
|
||||||
|
"""
|
||||||
|
db_path = SERVER_STATE["config"]["db_path"]
|
||||||
|
window_hours = SERVER_STATE["config"]["evaluation_window_hours"]
|
||||||
|
min_runs = SERVER_STATE["config"]["min_persistence_runs"]
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute("""
|
||||||
|
SELECT fingerprint, site_name, server, signature, severity, message, os_type, first_seen, last_seen, run_count, status
|
||||||
|
FROM active_issues
|
||||||
|
WHERE status = 'VERIFIED' AND run_count >= ?
|
||||||
|
""", (min_runs,))
|
||||||
|
rows = cursor.fetchall()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
report = []
|
||||||
|
for r in rows:
|
||||||
|
last_seen_dt = datetime.fromisoformat(r[8])
|
||||||
|
# Only return anomalies active within the evaluation window
|
||||||
|
if (now - last_seen_dt) <= timedelta(hours=window_hours):
|
||||||
|
report.append({
|
||||||
|
"fingerprint": r[0],
|
||||||
|
"site": r[1],
|
||||||
|
"server": r[2],
|
||||||
|
"signature": r[3],
|
||||||
|
"severity": r[4],
|
||||||
|
"message": r[5],
|
||||||
|
"os_type": r[6],
|
||||||
|
"first_seen": r[7],
|
||||||
|
"last_seen": r[8],
|
||||||
|
"consecutive_runs": r[9],
|
||||||
|
"evaluation_window": f"{window_hours}h",
|
||||||
|
"verified": True,
|
||||||
|
"status": r[10]
|
||||||
|
})
|
||||||
|
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/hermes/all")
|
||||||
|
def get_all_issues():
|
||||||
|
"""Diagnostic endpoint to inspect both transient candidate blips and verified anomalies."""
|
||||||
|
db_path = SERVER_STATE["config"]["db_path"]
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute("""
|
||||||
|
SELECT fingerprint, site_name, server, signature, severity, message, os_type, first_seen, last_seen, run_count, status
|
||||||
|
FROM active_issues
|
||||||
|
""")
|
||||||
|
rows = cursor.fetchall()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"fingerprint": r[0],
|
||||||
|
"site": r[1],
|
||||||
|
"server": r[2],
|
||||||
|
"signature": r[3],
|
||||||
|
"severity": r[4],
|
||||||
|
"message": r[5],
|
||||||
|
"os_type": r[6],
|
||||||
|
"first_seen": r[7],
|
||||||
|
"last_seen": r[8],
|
||||||
|
"run_count": r[9],
|
||||||
|
"status": r[10]
|
||||||
|
}
|
||||||
|
for r in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/health")
|
||||||
|
def health_check():
|
||||||
|
return {
|
||||||
|
"status": "healthy",
|
||||||
|
"server_name": SERVER_STATE["config"]["server_name"],
|
||||||
|
"fingerprint": SERVER_STATE["config"]["server_fingerprint"],
|
||||||
|
"tcp_port": SERVER_STATE["config"]["tcp_port"],
|
||||||
|
"hermes_port": SERVER_STATE["config"]["hermes_port"]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def run_server():
|
||||||
|
"""Runs the TCP socket listener and the Hermes REST API concurrently."""
|
||||||
|
config = SERVER_STATE["config"]
|
||||||
|
tcp_host = config["tcp_host"]
|
||||||
|
tcp_port = int(config["tcp_port"])
|
||||||
|
hermes_host = config["hermes_host"]
|
||||||
|
hermes_port = int(config["hermes_port"])
|
||||||
|
|
||||||
|
# Start TCP Socket Server
|
||||||
|
tcp_server = await asyncio.start_server(handle_socket_client, tcp_host, tcp_port)
|
||||||
|
print(f"[*] LOGAR TCP Socket Server listening on {tcp_host}:{tcp_port}")
|
||||||
|
|
||||||
|
# Start FastAPI / Uvicorn server for Hermes
|
||||||
|
uv_config = uvicorn.Config(app, host=hermes_host, port=hermes_port, log_level="warning")
|
||||||
|
uv_server = uvicorn.Server(uv_config)
|
||||||
|
print(f"[*] Hermes Reporting API available at http://{hermes_host}:{hermes_port}/api/hermes/report")
|
||||||
|
|
||||||
|
await asyncio.gather(
|
||||||
|
tcp_server.serve_forever(),
|
||||||
|
uv_server.serve()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="LOGAR Cloud Hub & TCP Socket Ingestion Server")
|
||||||
|
parser.add_argument("--config", default=CONFIG_FILE_NAME, help="Path to server_config.json")
|
||||||
|
parser.add_argument("--create-client-config", action="store_true", help="Generate a client config with encryption-only fingerprint and server address")
|
||||||
|
parser.add_argument("--client-out", default="client_config.json", help="Output file path for generated client config")
|
||||||
|
parser.add_argument("--server-host", default="127.0.0.1", help="Server address to embed in client config")
|
||||||
|
parser.add_argument("--server-port", type=int, default=None, help="TCP port to embed in client config")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
config = load_or_init_config(args.config)
|
||||||
|
init_db(config["db_path"])
|
||||||
|
|
||||||
|
# Load OpenPGP private key into memory
|
||||||
|
priv_key_obj, _ = pgpy.PGPKey.from_blob(config["private_key"])
|
||||||
|
SERVER_STATE["config"] = config
|
||||||
|
SERVER_STATE["private_key_obj"] = priv_key_obj
|
||||||
|
|
||||||
|
if args.create_client_config:
|
||||||
|
port = args.server_port or config["tcp_port"]
|
||||||
|
create_client_config(
|
||||||
|
server_host=args.server_host,
|
||||||
|
server_port=port,
|
||||||
|
output_path=args.client_out,
|
||||||
|
config_path=args.config
|
||||||
|
)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
print("=" * 60)
|
||||||
|
print(f" LOGAR Server Hub: {config['server_name']}")
|
||||||
|
print(f" Encryption Fingerprint: {config['server_fingerprint']}")
|
||||||
|
print(f" Evaluation Window: {config['evaluation_window_hours']} hours | Rule: {config['min_persistence_runs']}+ consecutive runs")
|
||||||
|
print("=" * 60)
|
||||||
|
|
||||||
|
try:
|
||||||
|
asyncio.run(run_server())
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
print("\n[!] Server shutting down.")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
+209
@@ -0,0 +1,209 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import socket
|
||||||
|
import struct
|
||||||
|
import argparse
|
||||||
|
import warnings
|
||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
|
||||||
|
# Suppress cryptography / pgpy deprecation notices
|
||||||
|
warnings.filterwarnings("ignore")
|
||||||
|
|
||||||
|
import pgpy
|
||||||
|
|
||||||
|
try:
|
||||||
|
import win32evtlog
|
||||||
|
except ImportError:
|
||||||
|
win32evtlog = None
|
||||||
|
|
||||||
|
CONFIG_FILE_NAME = "client_config.json"
|
||||||
|
|
||||||
|
|
||||||
|
def load_config(config_path: str = CONFIG_FILE_NAME):
|
||||||
|
if not os.path.exists(config_path):
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"Client configuration file not found at: {config_path}\n"
|
||||||
|
f"Generate one from the server using: python Server.py --create-client-config --client-out {config_path}"
|
||||||
|
)
|
||||||
|
with open(config_path, "r", encoding="utf-8") as f:
|
||||||
|
return json.load(f)
|
||||||
|
|
||||||
|
|
||||||
|
def get_machine_identifier() -> str:
|
||||||
|
"""
|
||||||
|
Returns the hostname of the machine sending the logs,
|
||||||
|
and appends the network/DNS domain if available.
|
||||||
|
"""
|
||||||
|
fqdn = socket.getfqdn()
|
||||||
|
if fqdn and "." in fqdn and not fqdn.startswith("localhost"):
|
||||||
|
return fqdn
|
||||||
|
|
||||||
|
hostname = socket.gethostname()
|
||||||
|
user_dns_domain = os.environ.get("USERDNSDOMAIN")
|
||||||
|
if user_dns_domain and user_dns_domain.lower() != hostname.lower():
|
||||||
|
return f"{hostname}.{user_dns_domain.lower()}"
|
||||||
|
|
||||||
|
try:
|
||||||
|
host_ip = socket.gethostbyname(hostname)
|
||||||
|
canonical_name = socket.gethostbyaddr(host_ip)[0]
|
||||||
|
if canonical_name and "." in canonical_name and not canonical_name.startswith("localhost"):
|
||||||
|
return canonical_name
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return hostname
|
||||||
|
|
||||||
|
|
||||||
|
def get_recent_windows_logs(hours: int = 6):
|
||||||
|
"""
|
||||||
|
Scans the Windows Application Event Log backwards for events within the window.
|
||||||
|
Edge Thinness & Noise Stripping: INFO and DEBUG events are dropped at the source.
|
||||||
|
"""
|
||||||
|
if win32evtlog is None:
|
||||||
|
print("[!] pywin32 is not installed or not running on Windows. Returning mock/empty candidate list.")
|
||||||
|
return []
|
||||||
|
|
||||||
|
server = "localhost"
|
||||||
|
log_type = "Application"
|
||||||
|
flags = win32evtlog.EVENTLOG_BACKWARDS_READ | win32evtlog.EVENTLOG_SEQUENTIAL_READ
|
||||||
|
|
||||||
|
try:
|
||||||
|
hand = win32evtlog.OpenEventLog(server, log_type)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[!] Error opening Windows event log: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
logs = []
|
||||||
|
cutoff_time = datetime.now() - timedelta(hours=hours)
|
||||||
|
machine_id = get_machine_identifier()
|
||||||
|
|
||||||
|
sev_map = {
|
||||||
|
1: "CRITICAL",
|
||||||
|
2: "ERROR",
|
||||||
|
3: "WARNING"
|
||||||
|
}
|
||||||
|
|
||||||
|
while True:
|
||||||
|
events = win32evtlog.ReadEventLog(hand, flags, 0)
|
||||||
|
if not events:
|
||||||
|
break
|
||||||
|
|
||||||
|
for event in events:
|
||||||
|
if event.TimeGenerated < cutoff_time:
|
||||||
|
break
|
||||||
|
|
||||||
|
# Drop conversational or informational noise (INFO=4, etc.) at source
|
||||||
|
# Only retain Critical (1), Error (2), and Warning (3)
|
||||||
|
if event.EventType in sev_map:
|
||||||
|
msg = " ".join(event.StringInserts) if event.StringInserts else "Event Log Entry"
|
||||||
|
logs.append({
|
||||||
|
"server": machine_id,
|
||||||
|
"os_type": "windows",
|
||||||
|
"signature": event.SourceName or "Windows-Event",
|
||||||
|
"severity": sev_map[event.EventType],
|
||||||
|
"message": msg[:2048] # Limit message length
|
||||||
|
})
|
||||||
|
|
||||||
|
if events[-1].TimeGenerated < cutoff_time:
|
||||||
|
break
|
||||||
|
|
||||||
|
win32evtlog.CloseEventLog(hand)
|
||||||
|
return logs
|
||||||
|
|
||||||
|
|
||||||
|
def send_encrypted_logs_over_socket(config: dict, logs: list):
|
||||||
|
"""
|
||||||
|
Encrypts the payload using the server's OpenPGP public key and streams
|
||||||
|
over an authenticated TCP socket. Zero local state is maintained on the client.
|
||||||
|
"""
|
||||||
|
server_host = config["server_host"]
|
||||||
|
server_port = int(config["server_port"])
|
||||||
|
auth_token = config["auth_token"]
|
||||||
|
pub_key_armored = config["server_public_key"]
|
||||||
|
expected_fp = config.get("server_fingerprint", "").replace(" ", "").upper()
|
||||||
|
|
||||||
|
# Load and verify server public key
|
||||||
|
pub_key, _ = pgpy.PGPKey.from_blob(pub_key_armored)
|
||||||
|
actual_fp = str(pub_key.fingerprint).replace(" ", "").upper()
|
||||||
|
if expected_fp and actual_fp != expected_fp:
|
||||||
|
raise ValueError(
|
||||||
|
f"Server fingerprint mismatch! Expected {expected_fp}, but key has {actual_fp}."
|
||||||
|
)
|
||||||
|
|
||||||
|
machine_id = get_machine_identifier()
|
||||||
|
|
||||||
|
# Prepare zero-state candidate batch
|
||||||
|
payload = {
|
||||||
|
"server": machine_id,
|
||||||
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"logs": logs
|
||||||
|
}
|
||||||
|
payload_json = json.dumps(payload)
|
||||||
|
|
||||||
|
# Encrypt payload with server's encryption-only key
|
||||||
|
pgp_msg = pgpy.PGPMessage.new(payload_json)
|
||||||
|
encrypted_msg = pub_key.encrypt(pgp_msg)
|
||||||
|
encrypted_armored = str(encrypted_msg)
|
||||||
|
|
||||||
|
# Envelope with socket authentication header
|
||||||
|
envelope = {
|
||||||
|
"auth_token": auth_token,
|
||||||
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"encrypted_payload": encrypted_armored
|
||||||
|
}
|
||||||
|
envelope_bytes = json.dumps(envelope).encode("utf-8")
|
||||||
|
|
||||||
|
# Connect over TCP socket and transmit with 4-byte length prefix framing
|
||||||
|
print(f"[*] Connecting to LOGAR server at {server_host}:{server_port} over secure TCP socket...")
|
||||||
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||||
|
sock.settimeout(15.0)
|
||||||
|
sock.connect((server_host, server_port))
|
||||||
|
|
||||||
|
# Send frame: length (4 bytes big-endian) + envelope
|
||||||
|
frame = struct.pack(">I", len(envelope_bytes)) + envelope_bytes
|
||||||
|
sock.sendall(frame)
|
||||||
|
|
||||||
|
# Receive response length
|
||||||
|
resp_len_bytes = sock.recv(4)
|
||||||
|
if not resp_len_bytes:
|
||||||
|
raise ConnectionError("Server closed connection without response.")
|
||||||
|
|
||||||
|
resp_len = struct.unpack(">I", resp_len_bytes)[0]
|
||||||
|
resp_bytes = bytearray()
|
||||||
|
while len(resp_bytes) < resp_len:
|
||||||
|
chunk = sock.recv(min(4096, resp_len - len(resp_bytes)))
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
resp_bytes.extend(chunk)
|
||||||
|
|
||||||
|
response = json.loads(resp_bytes.decode("utf-8"))
|
||||||
|
print(f"[+] Server response: {response}")
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="LOGAR Windows Edge Log Forwarder (Zero State)")
|
||||||
|
parser.add_argument("--config", default=CONFIG_FILE_NAME, help="Path to client_config.json")
|
||||||
|
parser.add_argument("--hours", type=int, default=6, help="Lookback window in hours for event logs")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
config = load_config(args.config)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[!] Configuration error: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print(f"[*] Scanning Windows Application event log for candidate anomalies (last {args.hours} hours)...")
|
||||||
|
candidate_logs = get_recent_windows_logs(hours=args.hours)
|
||||||
|
print(f"[*] Found {len(candidate_logs)} candidate anomalies (noise stripped at source).")
|
||||||
|
|
||||||
|
try:
|
||||||
|
send_encrypted_logs_over_socket(config, candidate_logs)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[!] Failed to stream logs to server: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Binary file not shown.
@@ -0,0 +1,194 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import socket
|
||||||
|
import struct
|
||||||
|
import argparse
|
||||||
|
import subprocess
|
||||||
|
import warnings
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
# Suppress cryptography / pgpy deprecation notices
|
||||||
|
warnings.filterwarnings("ignore")
|
||||||
|
|
||||||
|
import pgpy
|
||||||
|
|
||||||
|
CONFIG_FILE_NAME = "client_config.json"
|
||||||
|
|
||||||
|
|
||||||
|
def load_config(config_path: str = CONFIG_FILE_NAME):
|
||||||
|
if not os.path.exists(config_path):
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"Client configuration file not found at: {config_path}\n"
|
||||||
|
f"Generate one from the server using: python Server.py --create-client-config --client-out {config_path}"
|
||||||
|
)
|
||||||
|
with open(config_path, "r", encoding="utf-8") as f:
|
||||||
|
return json.load(f)
|
||||||
|
|
||||||
|
|
||||||
|
def get_machine_identifier() -> str:
|
||||||
|
"""
|
||||||
|
Returns the hostname of the machine sending the logs,
|
||||||
|
and appends the network/DNS domain if available.
|
||||||
|
"""
|
||||||
|
fqdn = socket.getfqdn()
|
||||||
|
if fqdn and "." in fqdn and not fqdn.startswith("localhost"):
|
||||||
|
return fqdn
|
||||||
|
|
||||||
|
hostname = socket.gethostname()
|
||||||
|
|
||||||
|
try:
|
||||||
|
if os.path.exists("/etc/resolv.conf"):
|
||||||
|
with open("/etc/resolv.conf", "r", encoding="utf-8") as f:
|
||||||
|
for line in f:
|
||||||
|
parts = line.strip().split()
|
||||||
|
if parts and parts[0] in ["domain", "search"] and len(parts) > 1:
|
||||||
|
domain = parts[1]
|
||||||
|
if domain and not domain.startswith("."):
|
||||||
|
return f"{hostname}.{domain}"
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
host_ip = socket.gethostbyname(hostname)
|
||||||
|
canonical_name = socket.gethostbyaddr(host_ip)[0]
|
||||||
|
if canonical_name and "." in canonical_name and not canonical_name.startswith("localhost"):
|
||||||
|
return canonical_name
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return hostname
|
||||||
|
|
||||||
|
|
||||||
|
def get_recent_linux_logs(hours: int = 6):
|
||||||
|
"""
|
||||||
|
Collects warnings and errors from systemd journalctl over the lookback window.
|
||||||
|
Edge Thinness: Drops INFO and DEBUG entries at the source.
|
||||||
|
"""
|
||||||
|
cmd = ["journalctl", "--since", f"{hours} hours ago", "-p", "warning", "--output=json"]
|
||||||
|
try:
|
||||||
|
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||||
|
except FileNotFoundError:
|
||||||
|
print("[!] journalctl command not found. Ensure this script runs on a systemd-enabled Linux system.")
|
||||||
|
return []
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[!] Error running journalctl: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
logs = []
|
||||||
|
machine_id = get_machine_identifier()
|
||||||
|
|
||||||
|
for line in result.stdout.splitlines():
|
||||||
|
line_str = line.strip()
|
||||||
|
if not line_str:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
entry = json.loads(line_str)
|
||||||
|
priority = str(entry.get("PRIORITY", "4"))
|
||||||
|
if int(priority) > 4:
|
||||||
|
continue
|
||||||
|
|
||||||
|
sev = "WARNING" if priority == "4" else "ERROR"
|
||||||
|
logs.append({
|
||||||
|
"server": machine_id,
|
||||||
|
"os_type": "linux",
|
||||||
|
"signature": entry.get("SYSLOG_IDENTIFIER", "unknown"),
|
||||||
|
"severity": sev,
|
||||||
|
"message": entry.get("MESSAGE", "")[:2048]
|
||||||
|
})
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
continue
|
||||||
|
|
||||||
|
return logs
|
||||||
|
|
||||||
|
|
||||||
|
def send_encrypted_logs_over_socket(config: dict, logs: list):
|
||||||
|
"""
|
||||||
|
Encrypts the payload using the server's OpenPGP public key and streams
|
||||||
|
over an authenticated TCP socket. Zero local state is maintained on the client.
|
||||||
|
"""
|
||||||
|
server_host = config["server_host"]
|
||||||
|
server_port = int(config["server_port"])
|
||||||
|
auth_token = config["auth_token"]
|
||||||
|
pub_key_armored = config["server_public_key"]
|
||||||
|
expected_fp = config.get("server_fingerprint", "").replace(" ", "").upper()
|
||||||
|
|
||||||
|
pub_key, _ = pgpy.PGPKey.from_blob(pub_key_armored)
|
||||||
|
actual_fp = str(pub_key.fingerprint).replace(" ", "").upper()
|
||||||
|
if expected_fp and actual_fp != expected_fp:
|
||||||
|
raise ValueError(
|
||||||
|
f"Server fingerprint mismatch! Expected {expected_fp}, but key has {actual_fp}."
|
||||||
|
)
|
||||||
|
|
||||||
|
machine_id = get_machine_identifier()
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"server": machine_id,
|
||||||
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"logs": logs
|
||||||
|
}
|
||||||
|
payload_json = json.dumps(payload)
|
||||||
|
|
||||||
|
pgp_msg = pgpy.PGPMessage.new(payload_json)
|
||||||
|
encrypted_msg = pub_key.encrypt(pgp_msg)
|
||||||
|
encrypted_armored = str(encrypted_msg)
|
||||||
|
|
||||||
|
envelope = {
|
||||||
|
"auth_token": auth_token,
|
||||||
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"encrypted_payload": encrypted_armored
|
||||||
|
}
|
||||||
|
envelope_bytes = json.dumps(envelope).encode("utf-8")
|
||||||
|
|
||||||
|
print(f"[*] Connecting to LOGAR server at {server_host}:{server_port} over secure TCP socket...")
|
||||||
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||||
|
sock.settimeout(15.0)
|
||||||
|
sock.connect((server_host, server_port))
|
||||||
|
|
||||||
|
frame = struct.pack(">I", len(envelope_bytes)) + envelope_bytes
|
||||||
|
sock.sendall(frame)
|
||||||
|
|
||||||
|
resp_len_bytes = sock.recv(4)
|
||||||
|
if not resp_len_bytes:
|
||||||
|
raise ConnectionError("Server closed connection without response.")
|
||||||
|
|
||||||
|
resp_len = struct.unpack(">I", resp_len_bytes)[0]
|
||||||
|
resp_bytes = bytearray()
|
||||||
|
while len(resp_bytes) < resp_len:
|
||||||
|
chunk = sock.recv(min(4096, resp_len - len(resp_bytes)))
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
resp_bytes.extend(chunk)
|
||||||
|
|
||||||
|
response = json.loads(resp_bytes.decode("utf-8"))
|
||||||
|
print(f"[+] Server response: {response}")
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="LOGAR Linux Edge Log Forwarder (Zero State)")
|
||||||
|
parser.add_argument("--config", default=CONFIG_FILE_NAME, help="Path to client_config.json")
|
||||||
|
parser.add_argument("--hours", type=int, default=6, help="Lookback window in hours for journalctl logs")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
config = load_config(args.config)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[!] Configuration error: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
machine_id = get_machine_identifier()
|
||||||
|
print(f"[*] Edge Forwarder Node: {machine_id}")
|
||||||
|
print(f"[*] Scanning Linux journalctl for candidate anomalies (last {args.hours} hours)...")
|
||||||
|
candidate_logs = get_recent_linux_logs(hours=args.hours)
|
||||||
|
print(f"[*] Found {len(candidate_logs)} candidate anomalies (noise stripped at source).")
|
||||||
|
|
||||||
|
try:
|
||||||
|
send_encrypted_logs_over_socket(config, candidate_logs)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[!] Failed to stream logs to server: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
# LOGAR Linux Edge Forwarder
|
||||||
|
|
||||||
|
Lightweight edge log forwarder for Linux servers running systemd.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
- **Zero Local State**: No local SQLite database or state tracking on the edge server.
|
||||||
|
- **Edge Noise Stripping**: Strips conversational/informational noise (`INFO`, `DEBUG`) directly at the source via `journalctl -p warning`.
|
||||||
|
- **End-to-End OpenPGP Encryption**: Encrypts logs using the server's public key; decrypted exclusively on the cloud hub.
|
||||||
|
- **Authenticated TCP Socket**: Direct, low-overhead TCP streaming with token authentication.
|
||||||
|
- **No GPG Binary Required**: Pure-Python implementation (`pgpy` + `cryptography`).
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
```bash
|
||||||
|
python3 -m pip install -r requirements.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
Place `client_config.json` generated by the server (`Server.py --create-client-config`) in the same directory as `Linux_Client.py`.
|
||||||
|
|
||||||
|
## Running the Forwarder
|
||||||
|
```bash
|
||||||
|
python3 Linux_Client.py --hours 6
|
||||||
|
```
|
||||||
|
|
||||||
|
## Cron / Systemd Timer Deployment
|
||||||
|
### Option A: Cron Job (Every 3 hours)
|
||||||
|
```bash
|
||||||
|
0 */3 * * * cd /opt/logar && /usr/bin/python3 Linux_Client.py --hours 6 >> /var/log/logar_client.log 2>&1
|
||||||
|
```
|
||||||
|
|
||||||
|
### Option B: Systemd Service & Timer
|
||||||
|
1. Create `/etc/systemd/system/logar-forwarder.service`:
|
||||||
|
```ini
|
||||||
|
[Unit]
|
||||||
|
Description=LOGAR Edge Forwarder
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/opt/logar
|
||||||
|
ExecStart=/usr/bin/python3 /opt/logar/Linux_Client.py --hours 6
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Create `/etc/systemd/system/logar-forwarder.timer`:
|
||||||
|
```ini
|
||||||
|
[Unit]
|
||||||
|
Description=Run LOGAR Edge Forwarder every 3 hours
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=5min
|
||||||
|
OnUnitActiveSec=3h
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Enable and start:
|
||||||
|
```bash
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
sudo systemctl enable --now logar-forwarder.timer
|
||||||
|
```
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Build script to compile Linux_Client into a standalone native ELF binary on Linux
|
||||||
|
set -e
|
||||||
|
|
||||||
|
echo "[*] Installing build requirements..."
|
||||||
|
pip3 install pyinstaller pgpy cryptography standard-imghdr
|
||||||
|
|
||||||
|
echo "[*] Compiling Linux_Client native binary..."
|
||||||
|
pyinstaller --onefile --clean --name Linux_Client.bin Linux_Client.py
|
||||||
|
|
||||||
|
echo "[+] Compilation successful: dist/Linux_Client.bin"
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"server_host": "192.168.1.100",
|
||||||
|
"server_port": 9443,
|
||||||
|
"server_fingerprint": "PASTE_SERVER_FINGERPRINT_HERE",
|
||||||
|
"server_public_key": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n...\n-----END PGP PUBLIC KEY BLOCK-----\n",
|
||||||
|
"auth_token": "PASTE_AUTH_TOKEN_HERE",
|
||||||
|
"site_name": "Frankfurt-DC"
|
||||||
|
}
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
pgpy>=0.6.0
|
||||||
|
standard-imghdr>=3.13.0; python_version >= "3.13"
|
||||||
|
cryptography>=42.0.0
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import struct
|
||||||
|
import unittest
|
||||||
|
import warnings
|
||||||
|
|
||||||
|
warnings.filterwarnings("ignore")
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||||
|
|
||||||
|
import Linux_Client
|
||||||
|
import pgpy
|
||||||
|
from pgpy.constants import PubKeyAlgorithm, KeyFlags, HashAlgorithm, SymmetricKeyAlgorithm, CompressionAlgorithm
|
||||||
|
|
||||||
|
|
||||||
|
class TestLinuxClientComponent(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.dummy_config = "test_linux_client_config.json"
|
||||||
|
# Generate dummy PGP key for testing
|
||||||
|
key = pgpy.PGPKey.new(PubKeyAlgorithm.RSAEncryptOrSign, 2048)
|
||||||
|
uid = pgpy.PGPUID.new("TestHub")
|
||||||
|
key.add_uid(
|
||||||
|
uid,
|
||||||
|
usage={KeyFlags.EncryptCommunications, KeyFlags.EncryptStorage},
|
||||||
|
hashes=[HashAlgorithm.SHA256],
|
||||||
|
ciphers=[SymmetricKeyAlgorithm.AES256],
|
||||||
|
compression=[CompressionAlgorithm.Uncompressed]
|
||||||
|
)
|
||||||
|
self.server_priv = key
|
||||||
|
self.server_pub = key.pubkey
|
||||||
|
self.fingerprint = str(key.pubkey.fingerprint)
|
||||||
|
|
||||||
|
with open(self.dummy_config, "w", encoding="utf-8") as f:
|
||||||
|
json.dump({
|
||||||
|
"server_host": "127.0.0.1",
|
||||||
|
"server_port": 9443,
|
||||||
|
"server_fingerprint": self.fingerprint,
|
||||||
|
"server_public_key": str(self.server_pub),
|
||||||
|
"auth_token": "secret-test-token"
|
||||||
|
}, f)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
if os.path.exists(self.dummy_config):
|
||||||
|
try:
|
||||||
|
os.remove(self.dummy_config)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def test_client_config_anonymity(self):
|
||||||
|
config = Linux_Client.load_config(self.dummy_config)
|
||||||
|
self.assertNotIn("server_name", config)
|
||||||
|
self.assertNotIn("name", config)
|
||||||
|
self.assertNotIn("site_name", config)
|
||||||
|
self.assertEqual(config["server_fingerprint"], self.fingerprint)
|
||||||
|
|
||||||
|
def test_get_machine_identifier(self):
|
||||||
|
machine_id = Linux_Client.get_machine_identifier()
|
||||||
|
self.assertIsInstance(machine_id, str)
|
||||||
|
self.assertGreater(len(machine_id), 0)
|
||||||
|
self.assertNotEqual(machine_id, "localhost")
|
||||||
|
|
||||||
|
def test_journalctl_parsing_and_priority_filter(self):
|
||||||
|
sample_journal_lines = [
|
||||||
|
json.dumps({"PRIORITY": "3", "SYSLOG_IDENTIFIER": "sshd", "MESSAGE": "Failed password for root"}),
|
||||||
|
json.dumps({"PRIORITY": "4", "SYSLOG_IDENTIFIER": "systemd", "MESSAGE": "Unit entered failed state"}),
|
||||||
|
json.dumps({"PRIORITY": "6", "SYSLOG_IDENTIFIER": "cron", "MESSAGE": "Informational session opened"}),
|
||||||
|
]
|
||||||
|
logs = []
|
||||||
|
machine_id = Linux_Client.get_machine_identifier()
|
||||||
|
for line_str in sample_journal_lines:
|
||||||
|
entry = json.loads(line_str)
|
||||||
|
priority = str(entry.get("PRIORITY", "4"))
|
||||||
|
if int(priority) > 4:
|
||||||
|
continue
|
||||||
|
sev = "WARNING" if priority == "4" else "ERROR"
|
||||||
|
logs.append({
|
||||||
|
"server": machine_id,
|
||||||
|
"os_type": "linux",
|
||||||
|
"signature": entry.get("SYSLOG_IDENTIFIER", "unknown"),
|
||||||
|
"severity": sev,
|
||||||
|
"message": entry.get("MESSAGE", "")
|
||||||
|
})
|
||||||
|
|
||||||
|
# Priority 6 must be stripped (INFO noise)
|
||||||
|
self.assertEqual(len(logs), 2)
|
||||||
|
self.assertEqual(logs[0]["severity"], "ERROR")
|
||||||
|
self.assertEqual(logs[1]["severity"], "WARNING")
|
||||||
|
|
||||||
|
def test_encryption_and_decryption(self):
|
||||||
|
config = Linux_Client.load_config(self.dummy_config)
|
||||||
|
pub_key, _ = pgpy.PGPKey.from_blob(config["server_public_key"])
|
||||||
|
payload = {
|
||||||
|
"server": Linux_Client.get_machine_identifier(),
|
||||||
|
"logs": [{"signature": "kernel", "severity": "ERROR", "message": "Kernel panic - not syncing"}]
|
||||||
|
}
|
||||||
|
msg = pgpy.PGPMessage.new(json.dumps(payload))
|
||||||
|
enc = pub_key.encrypt(msg)
|
||||||
|
self.assertTrue(str(enc).startswith("-----BEGIN PGP MESSAGE-----"))
|
||||||
|
|
||||||
|
dec = self.server_priv.decrypt(enc)
|
||||||
|
restored = json.loads(dec.message)
|
||||||
|
self.assertEqual(restored["logs"][0]["signature"], "kernel")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
# LOGAR Server Hub
|
||||||
|
|
||||||
|
Central Python/TCP ingestion server for the LOGAR Log Analysis System.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
- **Zero External GPG Requirement**: Uses pure-Python OpenPGP (`pgpy` + `cryptography`), no native GnuPG binary needed.
|
||||||
|
- **First-Run Key & Config Auto-generation**: Generates OpenPGP keypairs, auth tokens, and `server_config.json` automatically on first launch.
|
||||||
|
- **Client Config Exporter**: Generates `client_config.json` bundles containing the server's encryption-only fingerprint and address.
|
||||||
|
- **Cloud-Side Temporal Persistence**: SQLite database tracking candidate anomalies over 12-hour evaluation windows.
|
||||||
|
- **4-Run Persistence Rule**: Filters out transient infrastructure blips, promoting issues to `VERIFIED` anomalies only after persisting across $\ge 4$ runs.
|
||||||
|
- **Agentic Hermes Endpoint**: REST API (`GET /api/hermes/report`) providing verified system artifacts for Hermes agent alerts.
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
```bash
|
||||||
|
pip install -r requirements.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
## Running the Server
|
||||||
|
```bash
|
||||||
|
# Starts both the TCP socket listener (port 9443) and the Hermes API (port 8443)
|
||||||
|
python Server.py
|
||||||
|
```
|
||||||
|
|
||||||
|
## Generating Client Configurations
|
||||||
|
To deploy edge forwarders, generate a client config file:
|
||||||
|
```bash
|
||||||
|
python Server.py --create-client-config --server-host <SERVER_IP_OR_DNS> --server-port 9443 --site-name "Frankfurt-DC" --client-out client_config.json
|
||||||
|
```
|
||||||
|
Copy the generated `client_config.json` into the deployment directory of `Win_Client.py` or `Linux_Client.py`.
|
||||||
|
|
||||||
|
## Hermes Agent Integration
|
||||||
|
Hermes queries the verified anomalies via:
|
||||||
|
```
|
||||||
|
GET http://<SERVER_IP>:8443/api/hermes/report
|
||||||
|
```
|
||||||
|
Only issues meeting the 4-run persistence rule within the active 12-hour evaluation window are returned.
|
||||||
Binary file not shown.
@@ -0,0 +1,437 @@
|
|||||||
|
# Copy of Server.py without site_name in client_config
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import uuid
|
||||||
|
import struct
|
||||||
|
import socket
|
||||||
|
import sqlite3
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import secrets
|
||||||
|
import warnings
|
||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
from typing import Dict, Any, List, Optional
|
||||||
|
|
||||||
|
# Suppress cryptography / pgpy deprecation notices for a clean terminal output
|
||||||
|
warnings.filterwarnings("ignore")
|
||||||
|
|
||||||
|
import pgpy
|
||||||
|
from pgpy.constants import (
|
||||||
|
PubKeyAlgorithm,
|
||||||
|
KeyFlags,
|
||||||
|
HashAlgorithm,
|
||||||
|
SymmetricKeyAlgorithm,
|
||||||
|
CompressionAlgorithm
|
||||||
|
)
|
||||||
|
from fastapi import FastAPI, HTTPException
|
||||||
|
import uvicorn
|
||||||
|
|
||||||
|
CONFIG_FILE_NAME = "server_config.json"
|
||||||
|
DEFAULT_DB_FILE = "logar_state.db"
|
||||||
|
EVALUATION_WINDOW_HOURS = 12
|
||||||
|
RUN_THRESHOLD = 4
|
||||||
|
|
||||||
|
app = FastAPI(title="LOGAR Cloud Ingestion & Hermes Hub", version="2.0.0")
|
||||||
|
|
||||||
|
SERVER_STATE: Dict[str, Any] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def generate_server_keypair(server_name: str):
|
||||||
|
"""Generates an OpenPGP RSA 2048 key with encryption capability."""
|
||||||
|
key = pgpy.PGPKey.new(PubKeyAlgorithm.RSAEncryptOrSign, 2048)
|
||||||
|
uid = pgpy.PGPUID.new(server_name)
|
||||||
|
key.add_uid(
|
||||||
|
uid,
|
||||||
|
usage={KeyFlags.EncryptCommunications, KeyFlags.EncryptStorage},
|
||||||
|
hashes=[HashAlgorithm.SHA256],
|
||||||
|
ciphers=[SymmetricKeyAlgorithm.AES256],
|
||||||
|
compression=[CompressionAlgorithm.Uncompressed]
|
||||||
|
)
|
||||||
|
private_key_armored = str(key)
|
||||||
|
public_key_armored = str(key.pubkey)
|
||||||
|
fingerprint = str(key.pubkey.fingerprint)
|
||||||
|
return private_key_armored, public_key_armored, fingerprint
|
||||||
|
|
||||||
|
|
||||||
|
def load_or_init_config(config_path: str = CONFIG_FILE_NAME) -> Dict[str, Any]:
|
||||||
|
"""Loads existing server_config.json or creates a new one on first run."""
|
||||||
|
if os.path.exists(config_path):
|
||||||
|
print(f"[*] Loading server configuration from: {os.path.abspath(config_path)}")
|
||||||
|
with open(config_path, "r", encoding="utf-8") as f:
|
||||||
|
config = json.load(f)
|
||||||
|
return config
|
||||||
|
|
||||||
|
print(f"[!] Config '{config_path}' not found. Initializing first-run configuration...")
|
||||||
|
server_name = "LOGAR-Cloud-Hub"
|
||||||
|
private_key, public_key, fingerprint = generate_server_keypair(server_name)
|
||||||
|
auth_token = secrets.token_hex(24)
|
||||||
|
|
||||||
|
config = {
|
||||||
|
"server_name": server_name,
|
||||||
|
"tcp_host": "0.0.0.0",
|
||||||
|
"tcp_port": 9443,
|
||||||
|
"hermes_host": "0.0.0.0",
|
||||||
|
"hermes_port": 8443,
|
||||||
|
"auth_token": auth_token,
|
||||||
|
"db_path": DEFAULT_DB_FILE,
|
||||||
|
"evaluation_window_hours": EVALUATION_WINDOW_HOURS,
|
||||||
|
"min_persistence_runs": RUN_THRESHOLD,
|
||||||
|
"server_fingerprint": fingerprint,
|
||||||
|
"public_key": public_key,
|
||||||
|
"private_key": private_key
|
||||||
|
}
|
||||||
|
|
||||||
|
with open(config_path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(config, f, indent=2)
|
||||||
|
|
||||||
|
print(f"[+] Successfully generated new server config and OpenPGP keypair.")
|
||||||
|
print(f"[+] Server Encryption Fingerprint: {fingerprint}")
|
||||||
|
print(f"[+] Saved to: {os.path.abspath(config_path)}")
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
def create_client_config(
|
||||||
|
server_host: str,
|
||||||
|
server_port: int,
|
||||||
|
output_path: str,
|
||||||
|
config_path: str = CONFIG_FILE_NAME
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Creates a client configuration file containing the server address, auth token, and encryption-only key/fingerprint."""
|
||||||
|
server_conf = load_or_init_config(config_path)
|
||||||
|
|
||||||
|
client_conf = {
|
||||||
|
"server_host": server_host,
|
||||||
|
"server_port": server_port,
|
||||||
|
"server_fingerprint": server_conf["server_fingerprint"],
|
||||||
|
"server_public_key": server_conf["public_key"],
|
||||||
|
"auth_token": server_conf["auth_token"]
|
||||||
|
}
|
||||||
|
|
||||||
|
out_dir = os.path.dirname(os.path.abspath(output_path))
|
||||||
|
if out_dir and not os.path.exists(out_dir):
|
||||||
|
os.makedirs(out_dir, exist_ok=True)
|
||||||
|
|
||||||
|
with open(output_path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(client_conf, f, indent=2)
|
||||||
|
|
||||||
|
print(f"[+] Client configuration successfully written to: {os.path.abspath(output_path)}")
|
||||||
|
print(f" - Server Target: {server_host}:{server_port}")
|
||||||
|
print(f" - Encryption Fingerprint: {server_conf['server_fingerprint']}")
|
||||||
|
return client_conf
|
||||||
|
|
||||||
|
|
||||||
|
def init_db(db_path: str):
|
||||||
|
"""Initializes the SQLite schema for multi-run temporal tracking."""
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS active_issues (
|
||||||
|
fingerprint TEXT PRIMARY KEY,
|
||||||
|
site_name TEXT,
|
||||||
|
server TEXT,
|
||||||
|
signature TEXT,
|
||||||
|
severity TEXT,
|
||||||
|
message TEXT,
|
||||||
|
os_type TEXT,
|
||||||
|
first_seen TEXT,
|
||||||
|
last_seen TEXT,
|
||||||
|
run_count INTEGER,
|
||||||
|
status TEXT,
|
||||||
|
last_run_id TEXT
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS ingest_runs (
|
||||||
|
run_id TEXT PRIMARY KEY,
|
||||||
|
site_name TEXT,
|
||||||
|
server TEXT,
|
||||||
|
timestamp TEXT,
|
||||||
|
log_count INTEGER
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: int, min_runs: int) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Evaluates candidate issues against the 12-hour evaluation window and 4-run rule.
|
||||||
|
Zero-state clients send raw candidate entries; this engine handles temporal state.
|
||||||
|
"""
|
||||||
|
client_server = payload.get("server", "unknown-host")
|
||||||
|
site_name = payload.get("site_name") or (client_server.split(".", 1)[1] if "." in client_server else "default")
|
||||||
|
logs = payload.get("logs", [])
|
||||||
|
run_id = str(uuid.uuid4())
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
now_iso = now.isoformat()
|
||||||
|
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
cursor = conn.cursor()
|
||||||
|
|
||||||
|
cursor.execute(
|
||||||
|
"INSERT INTO ingest_runs (run_id, site_name, server, timestamp, log_count) VALUES (?, ?, ?, ?, ?)",
|
||||||
|
(run_id, site_name, client_server, now_iso, len(logs))
|
||||||
|
)
|
||||||
|
|
||||||
|
processed_count = 0
|
||||||
|
promoted_to_verified = 0
|
||||||
|
|
||||||
|
for log in logs:
|
||||||
|
severity = str(log.get("severity", "WARNING")).upper()
|
||||||
|
if severity in ["INFO", "DEBUG"]:
|
||||||
|
continue
|
||||||
|
|
||||||
|
signature = log.get("signature", "unknown")
|
||||||
|
server = log.get("server", client_server)
|
||||||
|
message = log.get("message", "")
|
||||||
|
os_type = log.get("os_type", "unknown")
|
||||||
|
fp = f"{site_name}:{server}:{signature}"
|
||||||
|
|
||||||
|
cursor.execute(
|
||||||
|
"SELECT run_count, first_seen, last_seen, status, last_run_id FROM active_issues WHERE fingerprint = ?",
|
||||||
|
(fp,)
|
||||||
|
)
|
||||||
|
row = cursor.fetchone()
|
||||||
|
|
||||||
|
if row:
|
||||||
|
run_count, first_seen_str, last_seen_str, current_status, last_run_id = row
|
||||||
|
try:
|
||||||
|
last_seen_dt = datetime.fromisoformat(last_seen_str)
|
||||||
|
except Exception:
|
||||||
|
last_seen_dt = now
|
||||||
|
|
||||||
|
if (now - last_seen_dt) > timedelta(hours=window_hours):
|
||||||
|
new_runs = 1
|
||||||
|
new_first_seen = now_iso
|
||||||
|
new_status = "TRANSIENT"
|
||||||
|
else:
|
||||||
|
if last_run_id != run_id:
|
||||||
|
new_runs = run_count + 1
|
||||||
|
else:
|
||||||
|
new_runs = run_count
|
||||||
|
new_first_seen = first_seen_str
|
||||||
|
new_status = "VERIFIED" if new_runs >= min_runs else "TRANSIENT"
|
||||||
|
|
||||||
|
if new_status == "VERIFIED" and current_status != "VERIFIED":
|
||||||
|
promoted_to_verified += 1
|
||||||
|
|
||||||
|
cursor.execute("""
|
||||||
|
UPDATE active_issues
|
||||||
|
SET run_count = ?, last_seen = ?, first_seen = ?, status = ?, last_run_id = ?, message = ?, severity = ?
|
||||||
|
WHERE fingerprint = ?
|
||||||
|
""", (new_runs, now_iso, new_first_seen, new_status, run_id, message, severity, fp))
|
||||||
|
else:
|
||||||
|
initial_status = "VERIFIED" if 1 >= min_runs else "TRANSIENT"
|
||||||
|
cursor.execute("""
|
||||||
|
INSERT INTO active_issues
|
||||||
|
(fingerprint, site_name, server, signature, severity, message, os_type, first_seen, last_seen, run_count, status, last_run_id)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""", (fp, site_name, server, signature, severity, message, os_type, now_iso, now_iso, 1, initial_status, run_id))
|
||||||
|
|
||||||
|
processed_count += 1
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
return {
|
||||||
|
"status": "success",
|
||||||
|
"run_id": run_id,
|
||||||
|
"processed": processed_count,
|
||||||
|
"promoted_verified": promoted_to_verified
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def handle_socket_client(reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
|
||||||
|
try:
|
||||||
|
length_bytes = await reader.readexactly(4)
|
||||||
|
length = struct.unpack(">I", length_bytes)[0]
|
||||||
|
if length <= 0 or length > 10 * 1024 * 1024:
|
||||||
|
raise ValueError(f"Invalid frame size: {length}")
|
||||||
|
|
||||||
|
payload_bytes = await reader.readexactly(length)
|
||||||
|
envelope = json.loads(payload_bytes.decode("utf-8"))
|
||||||
|
|
||||||
|
expected_token = SERVER_STATE["config"]["auth_token"]
|
||||||
|
provided_token = envelope.get("auth_token")
|
||||||
|
if not secrets.compare_digest(str(provided_token), str(expected_token)):
|
||||||
|
err_msg = json.dumps({"status": "error", "message": "Authentication failed"}).encode("utf-8")
|
||||||
|
writer.write(struct.pack(">I", len(err_msg)) + err_msg)
|
||||||
|
await writer.drain()
|
||||||
|
writer.close()
|
||||||
|
await writer.wait_closed()
|
||||||
|
return
|
||||||
|
|
||||||
|
encrypted_armored = envelope.get("encrypted_payload", "")
|
||||||
|
pgp_msg = pgpy.PGPMessage.from_blob(encrypted_armored)
|
||||||
|
priv_key = SERVER_STATE["private_key_obj"]
|
||||||
|
decrypted_obj = priv_key.decrypt(pgp_msg)
|
||||||
|
decrypted_json_str = decrypted_obj.message
|
||||||
|
log_payload = json.loads(decrypted_json_str)
|
||||||
|
|
||||||
|
res = process_ingested_logs(
|
||||||
|
log_payload,
|
||||||
|
db_path=SERVER_STATE["config"]["db_path"],
|
||||||
|
window_hours=SERVER_STATE["config"]["evaluation_window_hours"],
|
||||||
|
min_runs=SERVER_STATE["config"]["min_persistence_runs"]
|
||||||
|
)
|
||||||
|
|
||||||
|
resp_bytes = json.dumps(res).encode("utf-8")
|
||||||
|
writer.write(struct.pack(">I", len(resp_bytes)) + resp_bytes)
|
||||||
|
await writer.drain()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
err = json.dumps({"status": "error", "message": str(e)}).encode("utf-8")
|
||||||
|
try:
|
||||||
|
writer.write(struct.pack(">I", len(err)) + err)
|
||||||
|
await writer.drain()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
writer.close()
|
||||||
|
try:
|
||||||
|
await writer.wait_closed()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/hermes/report")
|
||||||
|
def get_hermes_report():
|
||||||
|
db_path = SERVER_STATE["config"]["db_path"]
|
||||||
|
window_hours = SERVER_STATE["config"]["evaluation_window_hours"]
|
||||||
|
min_runs = SERVER_STATE["config"]["min_persistence_runs"]
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute("""
|
||||||
|
SELECT fingerprint, site_name, server, signature, severity, message, os_type, first_seen, last_seen, run_count, status
|
||||||
|
FROM active_issues
|
||||||
|
WHERE status = 'VERIFIED' AND run_count >= ?
|
||||||
|
""", (min_runs,))
|
||||||
|
rows = cursor.fetchall()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
report = []
|
||||||
|
for r in rows:
|
||||||
|
last_seen_dt = datetime.fromisoformat(r[8])
|
||||||
|
if (now - last_seen_dt) <= timedelta(hours=window_hours):
|
||||||
|
report.append({
|
||||||
|
"fingerprint": r[0],
|
||||||
|
"site": r[1],
|
||||||
|
"server": r[2],
|
||||||
|
"signature": r[3],
|
||||||
|
"severity": r[4],
|
||||||
|
"message": r[5],
|
||||||
|
"os_type": r[6],
|
||||||
|
"first_seen": r[7],
|
||||||
|
"last_seen": r[8],
|
||||||
|
"consecutive_runs": r[9],
|
||||||
|
"evaluation_window": f"{window_hours}h",
|
||||||
|
"verified": True,
|
||||||
|
"status": r[10]
|
||||||
|
})
|
||||||
|
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/hermes/all")
|
||||||
|
def get_all_issues():
|
||||||
|
db_path = SERVER_STATE["config"]["db_path"]
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute("""
|
||||||
|
SELECT fingerprint, site_name, server, signature, severity, message, os_type, first_seen, last_seen, run_count, status
|
||||||
|
FROM active_issues
|
||||||
|
""")
|
||||||
|
rows = cursor.fetchall()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"fingerprint": r[0],
|
||||||
|
"site": r[1],
|
||||||
|
"server": r[2],
|
||||||
|
"signature": r[3],
|
||||||
|
"severity": r[4],
|
||||||
|
"message": r[5],
|
||||||
|
"os_type": r[6],
|
||||||
|
"first_seen": r[7],
|
||||||
|
"last_seen": r[8],
|
||||||
|
"run_count": r[9],
|
||||||
|
"status": r[10]
|
||||||
|
}
|
||||||
|
for r in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/health")
|
||||||
|
def health_check():
|
||||||
|
return {
|
||||||
|
"status": "healthy",
|
||||||
|
"server_name": SERVER_STATE["config"]["server_name"],
|
||||||
|
"fingerprint": SERVER_STATE["config"]["server_fingerprint"],
|
||||||
|
"tcp_port": SERVER_STATE["config"]["tcp_port"],
|
||||||
|
"hermes_port": SERVER_STATE["config"]["hermes_port"]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def run_server():
|
||||||
|
config = SERVER_STATE["config"]
|
||||||
|
tcp_host = config["tcp_host"]
|
||||||
|
tcp_port = int(config["tcp_port"])
|
||||||
|
hermes_host = config["hermes_host"]
|
||||||
|
hermes_port = int(config["hermes_port"])
|
||||||
|
|
||||||
|
tcp_server = await asyncio.start_server(handle_socket_client, tcp_host, tcp_port)
|
||||||
|
print(f"[*] LOGAR TCP Socket Server listening on {tcp_host}:{tcp_port}")
|
||||||
|
|
||||||
|
uv_config = uvicorn.Config(app, host=hermes_host, port=hermes_port, log_level="warning")
|
||||||
|
uv_server = uvicorn.Server(uv_config)
|
||||||
|
print(f"[*] Hermes Reporting API available at http://{hermes_host}:{hermes_port}/api/hermes/report")
|
||||||
|
|
||||||
|
await asyncio.gather(
|
||||||
|
tcp_server.serve_forever(),
|
||||||
|
uv_server.serve()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="LOGAR Cloud Hub & TCP Socket Ingestion Server")
|
||||||
|
parser.add_argument("--config", default=CONFIG_FILE_NAME, help="Path to server_config.json")
|
||||||
|
parser.add_argument("--create-client-config", action="store_true", help="Generate a client config with encryption-only fingerprint and server address")
|
||||||
|
parser.add_argument("--client-out", default="client_config.json", help="Output file path for generated client config")
|
||||||
|
parser.add_argument("--server-host", default="127.0.0.1", help="Server address to embed in client config")
|
||||||
|
parser.add_argument("--server-port", type=int, default=None, help="TCP port to embed in client config")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
config = load_or_init_config(args.config)
|
||||||
|
init_db(config["db_path"])
|
||||||
|
|
||||||
|
priv_key_obj, _ = pgpy.PGPKey.from_blob(config["private_key"])
|
||||||
|
SERVER_STATE["config"] = config
|
||||||
|
SERVER_STATE["private_key_obj"] = priv_key_obj
|
||||||
|
|
||||||
|
if args.create_client_config:
|
||||||
|
port = args.server_port or config["tcp_port"]
|
||||||
|
create_client_config(
|
||||||
|
server_host=args.server_host,
|
||||||
|
server_port=port,
|
||||||
|
output_path=args.client_out,
|
||||||
|
config_path=args.config
|
||||||
|
)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
print("=" * 60)
|
||||||
|
print(f" LOGAR Server Hub: {config['server_name']}")
|
||||||
|
print(f" Encryption Fingerprint: {config['server_fingerprint']}")
|
||||||
|
print(f" Evaluation Window: {config['evaluation_window_hours']} hours | Rule: {config['min_persistence_runs']}+ consecutive runs")
|
||||||
|
print("=" * 60)
|
||||||
|
|
||||||
|
try:
|
||||||
|
asyncio.run(run_server())
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
print("\n[!] Server shutting down.")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
pgpy>=0.6.0
|
||||||
|
standard-imghdr>=3.13.0; python_version >= "3.13"
|
||||||
|
cryptography>=42.0.0
|
||||||
|
fastapi>=0.110.0
|
||||||
|
uvicorn>=0.28.0
|
||||||
|
pydantic>=2.6.0
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"server_name": "LOGAR-Cloud-Hub",
|
||||||
|
"tcp_host": "0.0.0.0",
|
||||||
|
"tcp_port": 9443,
|
||||||
|
"hermes_host": "0.0.0.0",
|
||||||
|
"hermes_port": 8443,
|
||||||
|
"auth_token": "replace_with_secure_random_hex_token",
|
||||||
|
"db_path": "logar_state.db",
|
||||||
|
"evaluation_window_hours": 12,
|
||||||
|
"min_persistence_runs": 4,
|
||||||
|
"server_fingerprint": "AUTO_GENERATED_ON_FIRST_RUN",
|
||||||
|
"public_key": "AUTO_GENERATED_ON_FIRST_RUN",
|
||||||
|
"private_key": "AUTO_GENERATED_ON_FIRST_RUN"
|
||||||
|
}
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import socket
|
||||||
|
import struct
|
||||||
|
import sqlite3
|
||||||
|
import unittest
|
||||||
|
import urllib.request
|
||||||
|
import warnings
|
||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
|
||||||
|
warnings.filterwarnings("ignore")
|
||||||
|
|
||||||
|
# Ensure parent directory is in path to import Server
|
||||||
|
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||||
|
|
||||||
|
import Server
|
||||||
|
import pgpy
|
||||||
|
|
||||||
|
|
||||||
|
class TestServerComponent(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.test_db = "test_server_state.db"
|
||||||
|
self.test_config = "test_server_config.json"
|
||||||
|
if os.path.exists(self.test_db):
|
||||||
|
os.remove(self.test_db)
|
||||||
|
if os.path.exists(self.test_config):
|
||||||
|
os.remove(self.test_config)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
if os.path.exists(self.test_db):
|
||||||
|
try:
|
||||||
|
os.remove(self.test_db)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
if os.path.exists(self.test_config):
|
||||||
|
try:
|
||||||
|
os.remove(self.test_config)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def test_first_run_config_and_keypair_generation(self):
|
||||||
|
config = Server.load_or_init_config(self.test_config)
|
||||||
|
self.assertTrue(os.path.exists(self.test_config))
|
||||||
|
self.assertIn("server_fingerprint", config)
|
||||||
|
self.assertIn("public_key", config)
|
||||||
|
self.assertIn("private_key", config)
|
||||||
|
self.assertIn("auth_token", config)
|
||||||
|
self.assertNotIn("site_name", config)
|
||||||
|
|
||||||
|
# Verify keypair
|
||||||
|
priv_key, _ = pgpy.PGPKey.from_blob(config["private_key"])
|
||||||
|
pub_key, _ = pgpy.PGPKey.from_blob(config["public_key"])
|
||||||
|
self.assertEqual(str(pub_key.fingerprint), config["server_fingerprint"])
|
||||||
|
|
||||||
|
def test_create_client_config(self):
|
||||||
|
Server.load_or_init_config(self.test_config)
|
||||||
|
client_out = "test_client_out.json"
|
||||||
|
try:
|
||||||
|
client_conf = Server.create_client_config(
|
||||||
|
server_host="10.0.0.1",
|
||||||
|
server_port=9443,
|
||||||
|
output_path=client_out,
|
||||||
|
config_path=self.test_config
|
||||||
|
)
|
||||||
|
self.assertTrue(os.path.exists(client_out))
|
||||||
|
self.assertEqual(client_conf["server_host"], "10.0.0.1")
|
||||||
|
self.assertEqual(client_conf["server_port"], 9443)
|
||||||
|
# Verify no machine name or site_name is included
|
||||||
|
self.assertNotIn("server_name", client_conf)
|
||||||
|
self.assertNotIn("name", client_conf)
|
||||||
|
self.assertNotIn("site_name", client_conf)
|
||||||
|
finally:
|
||||||
|
if os.path.exists(client_out):
|
||||||
|
os.remove(client_out)
|
||||||
|
|
||||||
|
def test_4_run_rule_and_12h_window(self):
|
||||||
|
Server.init_db(self.test_db)
|
||||||
|
log_entry = {
|
||||||
|
"server": "app-worker-01.corp.local",
|
||||||
|
"signature": "PostgresConnTimeout",
|
||||||
|
"severity": "ERROR",
|
||||||
|
"message": "Connection to database pool timed out after 30s",
|
||||||
|
"os_type": "linux"
|
||||||
|
}
|
||||||
|
payload = {
|
||||||
|
"server": "app-worker-01.corp.local",
|
||||||
|
"logs": [log_entry]
|
||||||
|
}
|
||||||
|
|
||||||
|
# Runs 1 to 3: should remain TRANSIENT
|
||||||
|
for run_idx in range(1, 4):
|
||||||
|
res = Server.process_ingested_logs(payload, self.test_db, window_hours=12, min_runs=4)
|
||||||
|
self.assertEqual(res["status"], "success")
|
||||||
|
self.assertEqual(res["promoted_verified"], 0)
|
||||||
|
|
||||||
|
conn = sqlite3.connect(self.test_db)
|
||||||
|
c = conn.cursor()
|
||||||
|
c.execute("SELECT run_count, status FROM active_issues WHERE signature = ?", ("PostgresConnTimeout",))
|
||||||
|
row = c.fetchone()
|
||||||
|
conn.close()
|
||||||
|
self.assertEqual(row[0], 3)
|
||||||
|
self.assertEqual(row[1], "TRANSIENT")
|
||||||
|
|
||||||
|
# Run 4: promotes to VERIFIED!
|
||||||
|
res4 = Server.process_ingested_logs(payload, self.test_db, window_hours=12, min_runs=4)
|
||||||
|
self.assertEqual(res4["promoted_verified"], 1)
|
||||||
|
|
||||||
|
conn = sqlite3.connect(self.test_db)
|
||||||
|
c = conn.cursor()
|
||||||
|
c.execute("SELECT run_count, status FROM active_issues WHERE signature = ?", ("PostgresConnTimeout",))
|
||||||
|
row = c.fetchone()
|
||||||
|
conn.close()
|
||||||
|
self.assertEqual(row[0], 4)
|
||||||
|
self.assertEqual(row[1], "VERIFIED")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
# LOGAR Windows Edge Forwarder
|
||||||
|
|
||||||
|
Lightweight edge log forwarder for Windows servers.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
- **Zero Local State**: No local database or state tracking. Forwarder simply scans recent logs and streams candidates.
|
||||||
|
- **Edge Noise Stripping**: Strips conversational/informational noise (INFO, DEBUG, Audit) at the source.
|
||||||
|
- **End-to-End OpenPGP Encryption**: Encrypts logs using the server's public key so that only the server can decrypt them.
|
||||||
|
- **Authenticated TCP Socket**: Connects directly via raw TCP framing with token verification.
|
||||||
|
- **No GPG Binary Required**: Pure-Python cryptography (`pgpy` + `cryptography`).
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
```powershell
|
||||||
|
python -m pip install -r requirements.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
Place the `client_config.json` generated by the server (`Server.py --create-client-config`) in the same directory as `Win_Client.py`.
|
||||||
|
|
||||||
|
## Running the Forwarder
|
||||||
|
```powershell
|
||||||
|
python Win_Client.py --hours 6
|
||||||
|
```
|
||||||
|
|
||||||
|
## Scheduled Task Deployment
|
||||||
|
To run periodically via Windows Task Scheduler (e.g., every 3 hours):
|
||||||
|
```powershell
|
||||||
|
$Action = New-ScheduledTaskAction -Execute "python.exe" -Argument "C:\LOGAR\Win_Client.py --hours 6" -WorkingDirectory "C:\LOGAR"
|
||||||
|
$Trigger = New-ScheduledTaskTrigger -Once -At (Get-Date) -RepetitionInterval (New-TimeSpan -Hours 3)
|
||||||
|
Register-ScheduledTask -TaskName "LOGAR_Windows_Forwarder" -Action $Action -Trigger $Trigger -Description "LOGAR Edge Forwarder"
|
||||||
|
```
|
||||||
Binary file not shown.
@@ -0,0 +1,211 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import socket
|
||||||
|
import struct
|
||||||
|
import argparse
|
||||||
|
import warnings
|
||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
|
||||||
|
# Suppress cryptography / pgpy deprecation notices
|
||||||
|
warnings.filterwarnings("ignore")
|
||||||
|
|
||||||
|
import pgpy
|
||||||
|
|
||||||
|
try:
|
||||||
|
import win32evtlog
|
||||||
|
except ImportError:
|
||||||
|
win32evtlog = None
|
||||||
|
|
||||||
|
CONFIG_FILE_NAME = "client_config.json"
|
||||||
|
|
||||||
|
|
||||||
|
def load_config(config_path: str = CONFIG_FILE_NAME):
|
||||||
|
if not os.path.exists(config_path):
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"Client configuration file not found at: {config_path}\n"
|
||||||
|
f"Generate one from the server using: python Server.py --create-client-config --client-out {config_path}"
|
||||||
|
)
|
||||||
|
with open(config_path, "r", encoding="utf-8") as f:
|
||||||
|
return json.load(f)
|
||||||
|
|
||||||
|
|
||||||
|
def get_machine_identifier() -> str:
|
||||||
|
"""
|
||||||
|
Returns the hostname of the machine sending the logs,
|
||||||
|
and appends the network/DNS domain if available.
|
||||||
|
"""
|
||||||
|
fqdn = socket.getfqdn()
|
||||||
|
if fqdn and "." in fqdn and not fqdn.startswith("localhost"):
|
||||||
|
return fqdn
|
||||||
|
|
||||||
|
hostname = socket.gethostname()
|
||||||
|
user_dns_domain = os.environ.get("USERDNSDOMAIN")
|
||||||
|
if user_dns_domain and user_dns_domain.lower() != hostname.lower():
|
||||||
|
return f"{hostname}.{user_dns_domain.lower()}"
|
||||||
|
|
||||||
|
try:
|
||||||
|
host_ip = socket.gethostbyname(hostname)
|
||||||
|
canonical_name = socket.gethostbyaddr(host_ip)[0]
|
||||||
|
if canonical_name and "." in canonical_name and not canonical_name.startswith("localhost"):
|
||||||
|
return canonical_name
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return hostname
|
||||||
|
|
||||||
|
|
||||||
|
def get_recent_windows_logs(hours: int = 6):
|
||||||
|
"""
|
||||||
|
Scans the Windows Application Event Log backwards for events within the window.
|
||||||
|
Edge Thinness & Noise Stripping: INFO and DEBUG events are dropped at the source.
|
||||||
|
"""
|
||||||
|
if win32evtlog is None:
|
||||||
|
print("[!] pywin32 is not installed or not running on Windows. Returning mock/empty candidate list.")
|
||||||
|
return []
|
||||||
|
|
||||||
|
server = "localhost"
|
||||||
|
log_type = "Application"
|
||||||
|
flags = win32evtlog.EVENTLOG_BACKWARDS_READ | win32evtlog.EVENTLOG_SEQUENTIAL_READ
|
||||||
|
|
||||||
|
try:
|
||||||
|
hand = win32evtlog.OpenEventLog(server, log_type)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[!] Error opening Windows event log: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
logs = []
|
||||||
|
cutoff_time = datetime.now() - timedelta(hours=hours)
|
||||||
|
machine_id = get_machine_identifier()
|
||||||
|
|
||||||
|
sev_map = {
|
||||||
|
1: "CRITICAL",
|
||||||
|
2: "ERROR",
|
||||||
|
3: "WARNING"
|
||||||
|
}
|
||||||
|
|
||||||
|
while True:
|
||||||
|
events = win32evtlog.ReadEventLog(hand, flags, 0)
|
||||||
|
if not events:
|
||||||
|
break
|
||||||
|
|
||||||
|
for event in events:
|
||||||
|
if event.TimeGenerated < cutoff_time:
|
||||||
|
break
|
||||||
|
|
||||||
|
# Drop conversational or informational noise (INFO=4, etc.) at source
|
||||||
|
# Only retain Critical (1), Error (2), and Warning (3)
|
||||||
|
if event.EventType in sev_map:
|
||||||
|
msg = " ".join(event.StringInserts) if event.StringInserts else "Event Log Entry"
|
||||||
|
logs.append({
|
||||||
|
"server": machine_id,
|
||||||
|
"os_type": "windows",
|
||||||
|
"signature": event.SourceName or "Windows-Event",
|
||||||
|
"severity": sev_map[event.EventType],
|
||||||
|
"message": msg[:2048] # Limit message length
|
||||||
|
})
|
||||||
|
|
||||||
|
if events[-1].TimeGenerated < cutoff_time:
|
||||||
|
break
|
||||||
|
|
||||||
|
win32evtlog.CloseEventLog(hand)
|
||||||
|
return logs
|
||||||
|
|
||||||
|
|
||||||
|
def send_encrypted_logs_over_socket(config: dict, logs: list):
|
||||||
|
"""
|
||||||
|
Encrypts the payload using the server's OpenPGP public key and streams
|
||||||
|
over an authenticated TCP socket. Zero local state is maintained on the client.
|
||||||
|
"""
|
||||||
|
server_host = config["server_host"]
|
||||||
|
server_port = int(config["server_port"])
|
||||||
|
auth_token = config["auth_token"]
|
||||||
|
pub_key_armored = config["server_public_key"]
|
||||||
|
expected_fp = config.get("server_fingerprint", "").replace(" ", "").upper()
|
||||||
|
|
||||||
|
# Load and verify server public key
|
||||||
|
pub_key, _ = pgpy.PGPKey.from_blob(pub_key_armored)
|
||||||
|
actual_fp = str(pub_key.fingerprint).replace(" ", "").upper()
|
||||||
|
if expected_fp and actual_fp != expected_fp:
|
||||||
|
raise ValueError(
|
||||||
|
f"Server fingerprint mismatch! Expected {expected_fp}, but key has {actual_fp}."
|
||||||
|
)
|
||||||
|
|
||||||
|
machine_id = get_machine_identifier()
|
||||||
|
|
||||||
|
# Prepare zero-state candidate batch
|
||||||
|
payload = {
|
||||||
|
"server": machine_id,
|
||||||
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"logs": logs
|
||||||
|
}
|
||||||
|
payload_json = json.dumps(payload)
|
||||||
|
|
||||||
|
# Encrypt payload with server's encryption-only key
|
||||||
|
pgp_msg = pgpy.PGPMessage.new(payload_json)
|
||||||
|
encrypted_msg = pub_key.encrypt(pgp_msg)
|
||||||
|
encrypted_armored = str(encrypted_msg)
|
||||||
|
|
||||||
|
# Envelope with socket authentication header
|
||||||
|
envelope = {
|
||||||
|
"auth_token": auth_token,
|
||||||
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"encrypted_payload": encrypted_armored
|
||||||
|
}
|
||||||
|
envelope_bytes = json.dumps(envelope).encode("utf-8")
|
||||||
|
|
||||||
|
# Connect over TCP socket and transmit with 4-byte length prefix framing
|
||||||
|
print(f"[*] Connecting to LOGAR server at {server_host}:{server_port} over secure TCP socket...")
|
||||||
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||||
|
sock.settimeout(15.0)
|
||||||
|
sock.connect((server_host, server_port))
|
||||||
|
|
||||||
|
# Send frame: length (4 bytes big-endian) + envelope
|
||||||
|
frame = struct.pack(">I", len(envelope_bytes)) + envelope_bytes
|
||||||
|
sock.sendall(frame)
|
||||||
|
|
||||||
|
# Receive response length
|
||||||
|
resp_len_bytes = sock.recv(4)
|
||||||
|
if not resp_len_bytes:
|
||||||
|
raise ConnectionError("Server closed connection without response.")
|
||||||
|
|
||||||
|
resp_len = struct.unpack(">I", resp_len_bytes)[0]
|
||||||
|
resp_bytes = bytearray()
|
||||||
|
while len(resp_bytes) < resp_len:
|
||||||
|
chunk = sock.recv(min(4096, resp_len - len(resp_bytes)))
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
resp_bytes.extend(chunk)
|
||||||
|
|
||||||
|
response = json.loads(resp_bytes.decode("utf-8"))
|
||||||
|
print(f"[+] Server response: {response}")
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="LOGAR Windows Edge Log Forwarder (Zero State)")
|
||||||
|
parser.add_argument("--config", default=CONFIG_FILE_NAME, help="Path to client_config.json")
|
||||||
|
parser.add_argument("--hours", type=int, default=6, help="Lookback window in hours for event logs")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
config = load_config(args.config)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[!] Configuration error: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
machine_id = get_machine_identifier()
|
||||||
|
print(f"[*] Edge Forwarder Node: {machine_id}")
|
||||||
|
print(f"[*] Scanning Windows Application event log for candidate anomalies (last {args.hours} hours)...")
|
||||||
|
candidate_logs = get_recent_windows_logs(hours=args.hours)
|
||||||
|
print(f"[*] Found {len(candidate_logs)} candidate anomalies (noise stripped at source).")
|
||||||
|
|
||||||
|
try:
|
||||||
|
send_encrypted_logs_over_socket(config, candidate_logs)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[!] Failed to stream logs to server: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"server_host": "192.168.1.100",
|
||||||
|
"server_port": 9443,
|
||||||
|
"server_fingerprint": "PASTE_SERVER_FINGERPRINT_HERE",
|
||||||
|
"server_public_key": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n...\n-----END PGP PUBLIC KEY BLOCK-----\n",
|
||||||
|
"auth_token": "PASTE_AUTH_TOKEN_HERE",
|
||||||
|
"site_name": "Frankfurt-DC"
|
||||||
|
}
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
pgpy>=0.6.0
|
||||||
|
standard-imghdr>=3.13.0; python_version >= "3.13"
|
||||||
|
cryptography>=42.0.0
|
||||||
|
pywin32>=306
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import struct
|
||||||
|
import unittest
|
||||||
|
import warnings
|
||||||
|
|
||||||
|
warnings.filterwarnings("ignore")
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||||
|
|
||||||
|
import Win_Client
|
||||||
|
import pgpy
|
||||||
|
from pgpy.constants import PubKeyAlgorithm, KeyFlags, HashAlgorithm, SymmetricKeyAlgorithm, CompressionAlgorithm
|
||||||
|
|
||||||
|
|
||||||
|
class TestWinClientComponent(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.dummy_config = "test_win_client_config.json"
|
||||||
|
# Generate dummy PGP key for testing
|
||||||
|
key = pgpy.PGPKey.new(PubKeyAlgorithm.RSAEncryptOrSign, 2048)
|
||||||
|
uid = pgpy.PGPUID.new("TestHub")
|
||||||
|
key.add_uid(
|
||||||
|
uid,
|
||||||
|
usage={KeyFlags.EncryptCommunications, KeyFlags.EncryptStorage},
|
||||||
|
hashes=[HashAlgorithm.SHA256],
|
||||||
|
ciphers=[SymmetricKeyAlgorithm.AES256],
|
||||||
|
compression=[CompressionAlgorithm.Uncompressed]
|
||||||
|
)
|
||||||
|
self.server_priv = key
|
||||||
|
self.server_pub = key.pubkey
|
||||||
|
self.fingerprint = str(key.pubkey.fingerprint)
|
||||||
|
|
||||||
|
with open(self.dummy_config, "w", encoding="utf-8") as f:
|
||||||
|
json.dump({
|
||||||
|
"server_host": "127.0.0.1",
|
||||||
|
"server_port": 9443,
|
||||||
|
"server_fingerprint": self.fingerprint,
|
||||||
|
"server_public_key": str(self.server_pub),
|
||||||
|
"auth_token": "secret-test-token"
|
||||||
|
}, f)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
if os.path.exists(self.dummy_config):
|
||||||
|
try:
|
||||||
|
os.remove(self.dummy_config)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def test_client_config_anonymity(self):
|
||||||
|
config = Win_Client.load_config(self.dummy_config)
|
||||||
|
self.assertNotIn("server_name", config)
|
||||||
|
self.assertNotIn("name", config)
|
||||||
|
self.assertNotIn("site_name", config)
|
||||||
|
self.assertEqual(config["server_fingerprint"], self.fingerprint)
|
||||||
|
|
||||||
|
def test_get_machine_identifier(self):
|
||||||
|
machine_id = Win_Client.get_machine_identifier()
|
||||||
|
self.assertIsInstance(machine_id, str)
|
||||||
|
self.assertGreater(len(machine_id), 0)
|
||||||
|
self.assertNotEqual(machine_id, "localhost")
|
||||||
|
|
||||||
|
def test_encryption_and_envelope_creation(self):
|
||||||
|
config = Win_Client.load_config(self.dummy_config)
|
||||||
|
logs = [{
|
||||||
|
"server": Win_Client.get_machine_identifier(),
|
||||||
|
"signature": "TestWinSignature",
|
||||||
|
"severity": "WARNING",
|
||||||
|
"message": "Disk space threshold warning"
|
||||||
|
}]
|
||||||
|
|
||||||
|
pub_key, _ = pgpy.PGPKey.from_blob(config["server_public_key"])
|
||||||
|
payload = {
|
||||||
|
"server": Win_Client.get_machine_identifier(),
|
||||||
|
"logs": logs
|
||||||
|
}
|
||||||
|
msg = pgpy.PGPMessage.new(json.dumps(payload))
|
||||||
|
enc = pub_key.encrypt(msg)
|
||||||
|
self.assertTrue(str(enc).startswith("-----BEGIN PGP MESSAGE-----"))
|
||||||
|
|
||||||
|
# Decrypt with private key to verify end-to-end payload integrity
|
||||||
|
dec = self.server_priv.decrypt(enc)
|
||||||
|
restored = json.loads(dec.message)
|
||||||
|
self.assertEqual(restored["logs"][0]["signature"], "TestWinSignature")
|
||||||
|
|
||||||
|
def test_framing_protocol(self):
|
||||||
|
envelope_data = json.dumps({"test": "data"}).encode("utf-8")
|
||||||
|
frame = struct.pack(">I", len(envelope_data)) + envelope_data
|
||||||
|
self.assertEqual(len(frame), 4 + len(envelope_data))
|
||||||
|
length = struct.unpack(">I", frame[:4])[0]
|
||||||
|
self.assertEqual(length, len(envelope_data))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
pgpy>=0.6.0
|
||||||
|
standard-imghdr>=3.13.0; python_version >= "3.13"
|
||||||
|
cryptography>=42.0.0
|
||||||
|
fastapi>=0.110.0
|
||||||
|
uvicorn>=0.28.0
|
||||||
|
pydantic>=2.6.0
|
||||||
|
pywin32>=306; sys_platform == "win32"
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
import socket
|
||||||
|
import struct
|
||||||
|
import sqlite3
|
||||||
|
import urllib.request
|
||||||
|
import warnings
|
||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
|
||||||
|
warnings.filterwarnings("ignore")
|
||||||
|
import pgpy
|
||||||
|
|
||||||
|
# Test server endpoints
|
||||||
|
TCP_HOST = "127.0.0.1"
|
||||||
|
TCP_PORT = 9443
|
||||||
|
HERMES_HOST = "127.0.0.1"
|
||||||
|
HERMES_PORT = 8443
|
||||||
|
|
||||||
|
def run_tests():
|
||||||
|
print("=== [1] Verifying server_config.json & client_config.json ===")
|
||||||
|
assert os.path.exists("server_config.json"), "server_config.json must exist"
|
||||||
|
assert os.path.exists("client_config.json"), "client_config.json must exist"
|
||||||
|
|
||||||
|
with open("client_config.json", "r", encoding="utf-8") as f:
|
||||||
|
client_conf = json.load(f)
|
||||||
|
|
||||||
|
with open("server_config.json", "r", encoding="utf-8") as f:
|
||||||
|
server_conf = json.load(f)
|
||||||
|
|
||||||
|
assert "server_name" not in client_conf, "client_config.json must NOT contain server_name"
|
||||||
|
assert "name" not in client_conf, "client_config.json must NOT contain name"
|
||||||
|
assert "site_name" not in client_conf, "client_config.json must NOT contain site_name"
|
||||||
|
assert client_conf["server_fingerprint"] == server_conf["server_fingerprint"], "Fingerprints must match"
|
||||||
|
print(f"[OK] Verified client_config.json contains no machine/server/site name.")
|
||||||
|
print(f"[OK] Fingerprint verified: {client_conf['server_fingerprint']}")
|
||||||
|
|
||||||
|
# Load public key
|
||||||
|
pub_key, _ = pgpy.PGPKey.from_blob(client_conf["server_public_key"])
|
||||||
|
|
||||||
|
def send_socket_batch(logs, auth_token=client_conf["auth_token"]):
|
||||||
|
payload = {
|
||||||
|
"server": "test-edge-node.corp.internal",
|
||||||
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"logs": logs
|
||||||
|
}
|
||||||
|
pgp_msg = pgpy.PGPMessage.new(json.dumps(payload))
|
||||||
|
enc = pub_key.encrypt(pgp_msg)
|
||||||
|
|
||||||
|
envelope = {
|
||||||
|
"auth_token": auth_token,
|
||||||
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"encrypted_payload": str(enc)
|
||||||
|
}
|
||||||
|
envelope_bytes = json.dumps(envelope).encode("utf-8")
|
||||||
|
|
||||||
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||||
|
s.settimeout(5.0)
|
||||||
|
s.connect((TCP_HOST, TCP_PORT))
|
||||||
|
frame = struct.pack(">I", len(envelope_bytes)) + envelope_bytes
|
||||||
|
s.sendall(frame)
|
||||||
|
|
||||||
|
resp_len_bytes = s.recv(4)
|
||||||
|
resp_len = struct.unpack(">I", resp_len_bytes)[0]
|
||||||
|
resp_bytes = s.recv(resp_len)
|
||||||
|
return json.loads(resp_bytes.decode("utf-8"))
|
||||||
|
|
||||||
|
print("\n=== [2] Testing Socket Authentication Failure ===")
|
||||||
|
bad_resp = send_socket_batch([], auth_token="invalid-token-12345")
|
||||||
|
assert bad_resp.get("status") == "error", f"Expected error, got: {bad_resp}"
|
||||||
|
print(f"[OK] Bad auth rejected correctly: {bad_resp['message']}")
|
||||||
|
|
||||||
|
test_signature = "TestServiceCrash"
|
||||||
|
candidate_log = [{
|
||||||
|
"server": "test-edge-node",
|
||||||
|
"os_type": "linux",
|
||||||
|
"signature": test_signature,
|
||||||
|
"severity": "ERROR",
|
||||||
|
"message": "Out of memory killer triggered"
|
||||||
|
}]
|
||||||
|
|
||||||
|
print("\n=== [3] Testing Temporal Persistence & 4-Run Rule ===")
|
||||||
|
for run_num in range(1, 5):
|
||||||
|
resp = send_socket_batch(candidate_log)
|
||||||
|
assert resp.get("status") == "success", f"Run {run_num} failed: {resp}"
|
||||||
|
print(f"[Run {run_num}/4] Ingested successfully. Promoted to verified: {resp.get('promoted_verified')}")
|
||||||
|
|
||||||
|
# Inspect SQLite database directly
|
||||||
|
conn = sqlite3.connect(server_conf.get("db_path", "logar_state.db"))
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute("SELECT run_count, status FROM active_issues WHERE signature = ?", (test_signature,))
|
||||||
|
row = cursor.fetchone()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
assert row is not None, "Issue not found in SQLite"
|
||||||
|
run_count, status = row
|
||||||
|
print(f"[DB Verification] Issue '{test_signature}' -> run_count: {run_count}, status: {status}")
|
||||||
|
assert run_count >= 4, f"Expected run_count >= 4, got {run_count}"
|
||||||
|
assert status == "VERIFIED", f"Expected status 'VERIFIED', got {status}"
|
||||||
|
print("[OK] 4-Run Rule verified: Transient issue promoted to VERIFIED anomaly!")
|
||||||
|
|
||||||
|
print("\n=== [4] Testing Hermes Reporting Endpoint (/api/hermes/report) ===")
|
||||||
|
req = urllib.request.Request(f"http://{HERMES_HOST}:{HERMES_PORT}/api/hermes/report")
|
||||||
|
with urllib.request.urlopen(req, timeout=5) as response:
|
||||||
|
assert response.status == 200, f"Expected 200, got {response.status}"
|
||||||
|
hermes_data = json.loads(response.read().decode("utf-8"))
|
||||||
|
|
||||||
|
print(f"[Hermes API] Returned {len(hermes_data)} verified anomalies:")
|
||||||
|
found_issue = False
|
||||||
|
for issue in hermes_data:
|
||||||
|
print(f" - Fingerprint: {issue['fingerprint']} | Consecutive Runs: {issue['consecutive_runs']} | Status: {issue['status']}")
|
||||||
|
if issue["signature"] == test_signature:
|
||||||
|
found_issue = True
|
||||||
|
assert issue["verified"] is True
|
||||||
|
assert issue["consecutive_runs"] >= 4
|
||||||
|
|
||||||
|
assert found_issue, f"Test issue {test_signature} should be in Hermes report"
|
||||||
|
print("[OK] Hermes reporting validated!")
|
||||||
|
|
||||||
|
print("\n=== [5] Testing Windows Client Script Integration ===")
|
||||||
|
from Win_Client import get_recent_windows_logs
|
||||||
|
win_logs = get_recent_windows_logs(hours=6)
|
||||||
|
print(f"[Win_Client] Successfully queried Windows logs: {len(win_logs)} candidate entries.")
|
||||||
|
|
||||||
|
print("\n==========================================")
|
||||||
|
print(" ALL VERIFICATION TESTS PASSED SUCCESSFULLY! ")
|
||||||
|
print("==========================================")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
run_tests()
|
||||||
Reference in New Issue
Block a user