me0nline/LOGAR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

Implement edge filtering, state tracking, clean out/ directory, and add Gitea CI workflow
CI Test Suite / Run Component Tests & Pipeline Verification (push) Successful in 1m40s
Details

Browse Source
This commit is contained in:
me0nline
2026-09-04 15:32:39 +02:00
parent e634b060df
commit 7052e68589
26 changed files with 1310 additions and 1449 deletions
Download Patch File Download Diff File Expand all files Collapse all files
out/linux_client/Linux_Client.py
-194
View File
@@ -1,194 +0,0 @@
import os
import sys
import json
import socket
import struct
import argparse
import subprocess
import warnings
from datetime import datetime, timezone
# Suppress cryptography / pgpy deprecation notices
warnings.filterwarnings("ignore")
import pgpy
CONFIG_FILE_NAME = "client_config.json"
def load_config(config_path: str = CONFIG_FILE_NAME):
if not os.path.exists(config_path):
raise FileNotFoundError(
f"Client configuration file not found at: {config_path}\n"
f"Generate one from the server using: python Server.py --create-client-config --client-out {config_path}"
)
with open(config_path, "r", encoding="utf-8") as f:
return json.load(f)
def get_machine_identifier() -> str:
"""
Returns the hostname of the machine sending the logs,
and appends the network/DNS domain if available.
"""
fqdn = socket.getfqdn()
if fqdn and "." in fqdn and not fqdn.startswith("localhost"):
return fqdn
hostname = socket.gethostname()
try:
if os.path.exists("/etc/resolv.conf"):
with open("/etc/resolv.conf", "r", encoding="utf-8") as f:
for line in f:
parts = line.strip().split()
if parts and parts[0] in ["domain", "search"] and len(parts) > 1:
domain = parts[1]
if domain and not domain.startswith("."):
return f"{hostname}.{domain}"
except Exception:
pass
try:
host_ip = socket.gethostbyname(hostname)
canonical_name = socket.gethostbyaddr(host_ip)[0]
if canonical_name and "." in canonical_name and not canonical_name.startswith("localhost"):
return canonical_name
except Exception:
pass
return hostname
def get_recent_linux_logs(hours: int = 6):
"""
Collects warnings and errors from systemd journalctl over the lookback window.
Edge Thinness: Drops INFO and DEBUG entries at the source.
"""
cmd = ["journalctl", "--since", f"{hours} hours ago", "-p", "warning", "--output=json"]
try:
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
except FileNotFoundError:
print("[!] journalctl command not found. Ensure this script runs on a systemd-enabled Linux system.")
return []
except Exception as e:
print(f"[!] Error running journalctl: {e}")
return []
logs = []
machine_id = get_machine_identifier()
for line in result.stdout.splitlines():
line_str = line.strip()
if not line_str:
continue
try:
entry = json.loads(line_str)
priority = str(entry.get("PRIORITY", "4"))
if int(priority) > 4:
continue
sev = "WARNING" if priority == "4" else "ERROR"
logs.append({
"server": machine_id,
"os_type": "linux",
"signature": entry.get("SYSLOG_IDENTIFIER", "unknown"),
"severity": sev,
"message": entry.get("MESSAGE", "")[:2048]
})
except (json.JSONDecodeError, ValueError):
continue
return logs
def send_encrypted_logs_over_socket(config: dict, logs: list):
"""
Encrypts the payload using the server's OpenPGP public key and streams
over an authenticated TCP socket. Zero local state is maintained on the client.
"""
server_host = config["server_host"]
server_port = int(config["server_port"])
auth_token = config["auth_token"]
pub_key_armored = config["server_public_key"]
expected_fp = config.get("server_fingerprint", "").replace(" ", "").upper()
pub_key, _ = pgpy.PGPKey.from_blob(pub_key_armored)
actual_fp = str(pub_key.fingerprint).replace(" ", "").upper()
if expected_fp and actual_fp != expected_fp:
raise ValueError(
f"Server fingerprint mismatch! Expected {expected_fp}, but key has {actual_fp}."
)
machine_id = get_machine_identifier()
payload = {
"server": machine_id,
"timestamp": datetime.now(timezone.utc).isoformat(),
"logs": logs
}
payload_json = json.dumps(payload)
pgp_msg = pgpy.PGPMessage.new(payload_json)
encrypted_msg = pub_key.encrypt(pgp_msg)
encrypted_armored = str(encrypted_msg)
envelope = {
"auth_token": auth_token,
"timestamp": datetime.now(timezone.utc).isoformat(),
"encrypted_payload": encrypted_armored
}
envelope_bytes = json.dumps(envelope).encode("utf-8")
print(f"[*] Connecting to LOGAR server at {server_host}:{server_port} over secure TCP socket...")
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.settimeout(15.0)
sock.connect((server_host, server_port))
frame = struct.pack(">I", len(envelope_bytes)) + envelope_bytes
sock.sendall(frame)
resp_len_bytes = sock.recv(4)
if not resp_len_bytes:
raise ConnectionError("Server closed connection without response.")
resp_len = struct.unpack(">I", resp_len_bytes)[0]
resp_bytes = bytearray()
while len(resp_bytes) < resp_len:
chunk = sock.recv(min(4096, resp_len - len(resp_bytes)))
if not chunk:
break
resp_bytes.extend(chunk)
response = json.loads(resp_bytes.decode("utf-8"))
print(f"[+] Server response: {response}")
return response
def main():
parser = argparse.ArgumentParser(description="LOGAR Linux Edge Log Forwarder (Zero State)")
parser.add_argument("--config", default=CONFIG_FILE_NAME, help="Path to client_config.json")
parser.add_argument("--hours", type=int, default=6, help="Lookback window in hours for journalctl logs")
args = parser.parse_args()
try:
config = load_config(args.config)
except Exception as e:
print(f"[!] Configuration error: {e}")
sys.exit(1)
machine_id = get_machine_identifier()
print(f"[*] Edge Forwarder Node: {machine_id}")
print(f"[*] Scanning Linux journalctl for candidate anomalies (last {args.hours} hours)...")
candidate_logs = get_recent_linux_logs(hours=args.hours)
print(f"[*] Found {len(candidate_logs)} candidate anomalies (noise stripped at source).")
try:
send_encrypted_logs_over_socket(config, candidate_logs)
except Exception as e:
print(f"[!] Failed to stream logs to server: {e}")
sys.exit(1)
if __name__ == "__main__":
main()
out/linux_client/README.md
+112 -25
View File
@@ -1,50 +1,111 @@
# LOGAR Linux Edge Forwarder
Lightweight edge log forwarder for Linux servers running systemd.
Standalone compiled binary distribution for Linux edge servers running systemd.
## Features
- **Zero Local State**: No local SQLite database or state tracking on the edge server.
- **Edge Noise Stripping**: Strips conversational/informational noise (`INFO`, `DEBUG`) directly at the source via `journalctl -p warning`.
- **End-to-End OpenPGP Encryption**: Encrypts logs using the server's public key; decrypted exclusively on the cloud hub.
- **Authenticated TCP Socket**: Direct, low-overhead TCP streaming with token authentication.
- **No GPG Binary Required**: Pure-Python implementation (`pgpy` + `cryptography`).
---
## Overview
`Linux_Client.bin` is a self-contained, pre-compiled executable that queries `systemd-journald` via `journalctl`, filters logs directly at the source, encrypts the payload using OpenPGP, and streams candidate events over an authenticated TCP socket to the central LOGAR hub.
### Key Capabilities
- **Pre-compiled & Dependency-Free**: Ships as a standalone executable binary (`Linux_Client.bin`). No Python environment, pip packages, or GnuPG binaries are required on the host.
- **Source-Level Filtering**: Retains events spanning `INFO`, `WARNING`, and `ERROR` (`journalctl -p info`). Drops debug noise (priority 7) and skips events older than 24 hours.
- **State Tracking & Deduplication**: Maintains persistent client state in `client_state.json` (tracking systemd journalctl cursors and microsecond timestamps) so every log record is forwarded exactly once without duplicates.
- **Fail-Safe State Commit**: State is committed only when the server returns a verified `success` response. In the event of a network outage, state remains unchanged and unsent events are retried automatically on the next run.
- **End-to-End Encryption**: Encrypts payloads using the server's OpenPGP public key before transmission.
---
## 1. Generating & Deploying the Configuration File
### Step 1: Generate `client_config.json` on the Server
Run the following command on your central LOGAR server to export a client bundle tailored for your environment:
## Installation
```bash
python3 -m pip install -r requirements.txt
python Server.py --create-client-config --server-host <SERVER_IP_OR_DNS> --server-port 9443 --client-out client_config.json
```
## Configuration
Place `client_config.json` generated by the server (`Server.py --create-client-config`) in the same directory as `Linux_Client.py`.
- Replace `<SERVER_IP_OR_DNS>` with the reachable IP address or FQDN of your central LOGAR server hub.
- Default TCP port is `9443`.
## Running the Forwarder
```bash
python3 Linux_Client.py --hours 6
### Step 2: Configuration Structure
The generated `client_config.json` contains:
```json
{
"server_host": "192.168.1.100",
"server_port": 9443,
"server_fingerprint": "375388960531264EA0648EC0D2C4E4ABC6F22AC2",
"server_public_key": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n...",
"auth_token": "a1b2c3d4e5f6..."
}
```
## Cron / Systemd Timer Deployment
### Option A: Cron Job (Every 3 hours)
> [!NOTE]
> A reference example is provided in `client_config.sample.json`. The configuration file contains **no host-specific names or site names** to ensure client anonymity and easy redistribution.
### Step 3: Copy to Edge Node
Place `Linux_Client.bin` and `client_config.json` into the target directory (recommended: `/opt/logar/`):
```bash
0 */3 * * * cd /opt/logar && /usr/bin/python3 Linux_Client.py --hours 6 >> /var/log/logar_client.log 2>&1
sudo mkdir -p /opt/logar
sudo cp Linux_Client.bin client_config.json /opt/logar/
sudo chmod +x /opt/logar/Linux_Client.bin
```
### Option B: Systemd Service & Timer
1. Create `/etc/systemd/system/logar-forwarder.service`:
---
## 2. Running Manually
Test the forwarder interactively:
```bash
cd /opt/logar
./Linux_Client.bin --hours 24
```
### Command-Line Arguments
| Argument | Default | Description |
| :--- | :--- | :--- |
| `--config` | `client_config.json` | Path to client configuration file |
| `--hours` | `24` | Lookback window in hours for journal logs |
| `--state-file` | `client_state.json` | Path to persistent state file |
| `--no-state` | `False` | Disable state tracking and send all events matching lookback window |
---
## 3. Installing as a Systemd Service & Timer (Recommended)
Running `Linux_Client.bin` via a systemd timer ensures reliable periodic execution, automatic restart, and native log integration with `journalctl`.
### Step 1: Create the Systemd Service Unit
Create `/etc/systemd/system/logar-forwarder.service`:
```ini
[Unit]
Description=LOGAR Edge Forwarder
After=network.target
Description=LOGAR Edge Log Forwarder
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
WorkingDirectory=/opt/logar
ExecStart=/usr/bin/python3 /opt/logar/Linux_Client.py --hours 6
ExecStart=/opt/logar/Linux_Client.bin --hours 24
User=root
StandardOutput=journal
StandardError=journal
[Install]
WantedBy=multi-user.target
```
2. Create `/etc/systemd/system/logar-forwarder.timer`:
### Step 2: Create the Systemd Timer Unit
Create `/etc/systemd/system/logar-forwarder.timer` to execute the forwarder every 3 hours (with a 5-minute initial delay upon boot):
```ini
[Unit]
Description=Run LOGAR Edge Forwarder every 3 hours
Description=Run LOGAR Edge Forwarder periodically
Requires=logar-forwarder.service
[Timer]
OnBootSec=5min
@@ -55,8 +116,34 @@ Persistent=true
WantedBy=timers.target
```
3. Enable and start:
### Step 3: Enable and Start the Timer
```bash
sudo systemctl daemon-reload
sudo systemctl enable --now logar-forwarder.timer
```
### Step 4: Verify Timer & Service Status
```bash
# Check timer schedule
sudo systemctl list-timers --all | grep logar
# Trigger an immediate manual execution
sudo systemctl start logar-forwarder.service
# View execution logs
sudo journalctl -u logar-forwarder.service -n 50
```
---
## 4. Alternative: Cron Job Deployment
If systemd timers are not preferred, configure a periodic cron job running every 3 hours:
```bash
# Open root crontab
sudo crontab -e
# Add the following entry:
0 */3 * * * cd /opt/logar && ./Linux_Client.bin --hours 24 >> /var/log/logar_forwarder.log 2>&1
```
out/linux_client/build_bin.sh
-11
View File
@@ -1,11 +0,0 @@
#!/usr/bin/env bash
# Build script to compile Linux_Client into a standalone native ELF binary on Linux
set -e
echo "[*] Installing build requirements..."
pip3 install pyinstaller pgpy cryptography standard-imghdr
echo "[*] Compiling Linux_Client native binary..."
pyinstaller --onefile --clean --name Linux_Client.bin Linux_Client.py
echo "[+] Compilation successful: dist/Linux_Client.bin"
out/linux_client/client_config.sample.json
+1 -2
View File
@@ -3,6 +3,5 @@
"server_port": 9443,
"server_fingerprint": "PASTE_SERVER_FINGERPRINT_HERE",
"server_public_key": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n...\n-----END PGP PUBLIC KEY BLOCK-----\n",
"auth_token": "PASTE_AUTH_TOKEN_HERE",
"site_name": "Frankfurt-DC"
"auth_token": "PASTE_AUTH_TOKEN_HERE"
}
out/linux_client/requirements.txt
-3
View File
@@ -1,3 +0,0 @@
pgpy>=0.6.0
standard-imghdr>=3.13.0; python_version >= "3.13"
cryptography>=42.0.0
out/linux_client/test/test_linux_client.py
-107
View File
@@ -1,107 +0,0 @@
import os
import sys
import json
import struct
import unittest
import warnings
warnings.filterwarnings("ignore")
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
import Linux_Client
import pgpy
from pgpy.constants import PubKeyAlgorithm, KeyFlags, HashAlgorithm, SymmetricKeyAlgorithm, CompressionAlgorithm
class TestLinuxClientComponent(unittest.TestCase):
def setUp(self):
self.dummy_config = "test_linux_client_config.json"
# Generate dummy PGP key for testing
key = pgpy.PGPKey.new(PubKeyAlgorithm.RSAEncryptOrSign, 2048)
uid = pgpy.PGPUID.new("TestHub")
key.add_uid(
uid,
usage={KeyFlags.EncryptCommunications, KeyFlags.EncryptStorage},
hashes=[HashAlgorithm.SHA256],
ciphers=[SymmetricKeyAlgorithm.AES256],
compression=[CompressionAlgorithm.Uncompressed]
)
self.server_priv = key
self.server_pub = key.pubkey
self.fingerprint = str(key.pubkey.fingerprint)
with open(self.dummy_config, "w", encoding="utf-8") as f:
json.dump({
"server_host": "127.0.0.1",
"server_port": 9443,
"server_fingerprint": self.fingerprint,
"server_public_key": str(self.server_pub),
"auth_token": "secret-test-token"
}, f)
def tearDown(self):
if os.path.exists(self.dummy_config):
try:
os.remove(self.dummy_config)
except Exception:
pass
def test_client_config_anonymity(self):
config = Linux_Client.load_config(self.dummy_config)
self.assertNotIn("server_name", config)
self.assertNotIn("name", config)
self.assertNotIn("site_name", config)
self.assertEqual(config["server_fingerprint"], self.fingerprint)
def test_get_machine_identifier(self):
machine_id = Linux_Client.get_machine_identifier()
self.assertIsInstance(machine_id, str)
self.assertGreater(len(machine_id), 0)
self.assertNotEqual(machine_id, "localhost")
def test_journalctl_parsing_and_priority_filter(self):
sample_journal_lines = [
json.dumps({"PRIORITY": "3", "SYSLOG_IDENTIFIER": "sshd", "MESSAGE": "Failed password for root"}),
json.dumps({"PRIORITY": "4", "SYSLOG_IDENTIFIER": "systemd", "MESSAGE": "Unit entered failed state"}),
json.dumps({"PRIORITY": "6", "SYSLOG_IDENTIFIER": "cron", "MESSAGE": "Informational session opened"}),
]
logs = []
machine_id = Linux_Client.get_machine_identifier()
for line_str in sample_journal_lines:
entry = json.loads(line_str)
priority = str(entry.get("PRIORITY", "4"))
if int(priority) > 4:
continue
sev = "WARNING" if priority == "4" else "ERROR"
logs.append({
"server": machine_id,
"os_type": "linux",
"signature": entry.get("SYSLOG_IDENTIFIER", "unknown"),
"severity": sev,
"message": entry.get("MESSAGE", "")
})
# Priority 6 must be stripped (INFO noise)
self.assertEqual(len(logs), 2)
self.assertEqual(logs[0]["severity"], "ERROR")
self.assertEqual(logs[1]["severity"], "WARNING")
def test_encryption_and_decryption(self):
config = Linux_Client.load_config(self.dummy_config)
pub_key, _ = pgpy.PGPKey.from_blob(config["server_public_key"])
payload = {
"server": Linux_Client.get_machine_identifier(),
"logs": [{"signature": "kernel", "severity": "ERROR", "message": "Kernel panic - not syncing"}]
}
msg = pgpy.PGPMessage.new(json.dumps(payload))
enc = pub_key.encrypt(msg)
self.assertTrue(str(enc).startswith("-----BEGIN PGP MESSAGE-----"))
dec = self.server_priv.decrypt(enc)
restored = json.loads(dec.message)
self.assertEqual(restored["logs"][0]["signature"], "kernel")
if __name__ == "__main__":
unittest.main()