Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8f8a605c98 | ||
|
|
548b2fd8ad | ||
|
|
9b80007a86 | ||
|
|
cd6b9baa8e | ||
|
|
ea2ff1c4a9 | ||
|
|
2d7357394f | ||
|
|
4c1ab1ea86 | ||
|
|
2ecb356516 | ||
|
|
3e7870c278 | ||
|
|
57eb05fab1 | ||
|
|
e41c304208 | ||
|
|
57f3dd896f | ||
|
|
303744a3f8 | ||
|
|
d9bc8df002 | ||
|
|
8b12186b43 | ||
|
|
df9f5d191b | ||
|
|
9ebfadbe89 | ||
|
|
fe9d43cab6 | ||
|
|
f0c4a82fc3 | ||
|
|
2c08f9fc15 | ||
|
|
f2a9e43619 | ||
|
|
909dc640e3 | ||
|
|
0edae4d797 | ||
|
|
cc25c31884 | ||
|
|
421eafb72d | ||
|
|
ca8b0153fa | ||
|
|
50f76ca3c3 | ||
|
|
ea6410d391 | ||
|
|
41d7d66a56 | ||
|
|
959f2977f0 | ||
|
|
40cdbf349c | ||
|
|
ca1b5b5d25 | ||
|
|
c1d4675321 | ||
|
|
2d33b14d60 | ||
|
|
2b0b680188 | ||
|
|
f0bf90e033 | ||
|
|
22072a0b88 | ||
|
|
730f0ba7ca | ||
|
|
b7c27ee3a1 | ||
|
|
fa662da476 | ||
|
|
6cb9631958 | ||
|
|
c03bb348b5 | ||
|
|
2d39f3c007 | ||
|
|
78e62b31d2 | ||
|
|
860e39b360 | ||
|
|
2a04244db3 | ||
|
|
9f999fc6c3 | ||
|
|
3399c6d57b | ||
|
|
c6d04f8896 | ||
|
|
0a67b43619 |
@@ -1,9 +1,11 @@
|
||||
name: Build and Push LOGAR Runner
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'runner/**'
|
||||
- '.gitea/workflows/build-runner.yml'
|
||||
workflow_dispatch:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
@@ -56,3 +58,14 @@ jobs:
|
||||
curl -s -X POST \
|
||||
"https://portainer.eibl.tech/api/stacks/webhooks/6e67c0b7-1c5d-4bb2-b8b8-4a6ec66d3b87"
|
||||
echo "✓ Portainer webhook triggered"
|
||||
|
||||
- name: Clean up temporary runner base images (node:16-bullseye) from Docker host
|
||||
if: always()
|
||||
run: |
|
||||
echo "[*] Cleaning up temporary base images from Docker host daemon..."
|
||||
# Remove node:16-bullseye images used by act_runner for ubuntu-latest
|
||||
docker images --filter "reference=*bullseye*" -q | xargs -r docker rmi -f 2>/dev/null || true
|
||||
docker rmi -f node:16-bullseye node:16-bullseye-slim 2>/dev/null || true
|
||||
# Prune dangling intermediate build cache layers
|
||||
docker image prune -f 2>/dev/null || true
|
||||
echo "[+] Cleaned up 16-bullseye and intermediate build images from host daemon."
|
||||
|
||||
@@ -16,17 +16,20 @@ jobs:
|
||||
name: Run Component Tests & Pipeline Verification
|
||||
if: "!startsWith(github.ref, 'refs/tags/')"
|
||||
runs-on: logar
|
||||
env:
|
||||
PYTHONPATH: .
|
||||
LOGAR_DISABLE_AUTO_UPDATE: "1"
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install Dependencies
|
||||
- name: Verify Environment & Dependencies
|
||||
run: |
|
||||
python3 compilation/ensure_pip.py
|
||||
python3 -m pip install --upgrade pip --break-system-packages 2>/dev/null || python3 -m pip install --upgrade pip 2>/dev/null || true
|
||||
python3 -m pip install -r compilation/requirements.txt --break-system-packages 2>/dev/null || python3 -m pip install -r compilation/requirements.txt
|
||||
python3 --version
|
||||
python3 -m pip install -r compilation/requirements.txt --no-warn-script-location --break-system-packages 2>/dev/null || python3 -m pip install -r compilation/requirements.txt --no-warn-script-location || true
|
||||
python3 -c "import pgpy, fastapi, uvicorn, websockets, sqlalchemy; print('[+] All required dependencies are verified.')"
|
||||
|
||||
- name: Verify Python Syntax
|
||||
run: |
|
||||
|
||||
@@ -23,44 +23,14 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install System and Cross-Compilation Build Dependencies
|
||||
- name: Verify Builder Runner Environment & Dependencies
|
||||
run: |
|
||||
echo "=== SYSTEM DIAGNOSTICS ==="
|
||||
uname -a || true
|
||||
python3 --version || true
|
||||
SUDO=""
|
||||
if [ "$(id -u)" -ne 0 ] && command -v sudo >/dev/null 2>&1; then
|
||||
SUDO="sudo"
|
||||
fi
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
echo "[*] Installing Python shared libraries, headers, NSIS via apt-get..."
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
$SUDO apt-get clean || true
|
||||
$SUDO apt-get update --allow-releaseinfo-change -o Acquire::Check-Valid-Until=false -y || $SUDO apt-get update -y || true
|
||||
$SUDO apt-get -f install -y || true
|
||||
PY_VER=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" 2>/dev/null || echo "3.9")
|
||||
echo "[*] Detected Python version: ${PY_VER}"
|
||||
$SUDO apt-get install -y --fix-broken --fix-missing --allow-unauthenticated --no-install-recommends \
|
||||
nsis nsis-common "libpython${PY_VER}" "libpython${PY_VER}-dev" "python${PY_VER}-dev" \
|
||||
python3-dev libpython3-dev build-essential gcc make curl tar gzip zip unzip || true
|
||||
$SUDO apt-get install -y --fix-broken --fix-missing --allow-unauthenticated --no-install-recommends nsis nsis-common || true
|
||||
for pkg in nsis nsis-common "libpython${PY_VER}" "libpython${PY_VER}-dev" "python${PY_VER}-dev" python3-dev libpython3-dev build-essential; do
|
||||
$SUDO apt-get install -y --fix-broken --fix-missing --allow-unauthenticated --no-install-recommends "$pkg" || true
|
||||
done
|
||||
echo "[*] Verifying makensis:"
|
||||
which makensis || echo "makensis not in PATH"
|
||||
echo "[*] Verifying libpython shared library:"
|
||||
find /usr/lib /usr/local/lib -name "libpython*.so*" 2>/dev/null || true
|
||||
elif command -v apk >/dev/null 2>&1; then
|
||||
echo "[*] Installing dependencies via apk..."
|
||||
$SUDO apk add --no-cache nsis python3-dev build-base gcc make curl tar gzip zip unzip || true
|
||||
elif command -v dnf >/dev/null 2>&1; then
|
||||
echo "[*] Installing dependencies via dnf..."
|
||||
$SUDO dnf install -y mingw32-nsis nsis python3-devel gcc make curl tar gzip zip unzip || true
|
||||
fi
|
||||
python3 compilation/ensure_pip.py
|
||||
python3 -m pip install --upgrade pip --break-system-packages 2>/dev/null || python3 -m pip install --upgrade pip 2>/dev/null || true
|
||||
python3 -m pip install pyinstaller distlib cryptography -r compilation/requirements.txt --break-system-packages 2>/dev/null || python3 -m pip install pyinstaller distlib cryptography -r compilation/requirements.txt
|
||||
echo "=== LOGAR BUILD RUNNER ENVIRONMENT ==="
|
||||
uname -a
|
||||
python3 --version
|
||||
which makensis || echo "[!] Warning: makensis not found in PATH"
|
||||
python3 -m pip install -r compilation/requirements.txt --no-warn-script-location --break-system-packages 2>/dev/null || python3 -m pip install -r compilation/requirements.txt --no-warn-script-location || true
|
||||
python3 -c "import PyInstaller, distlib, pgpy, fastapi, uvicorn, websockets, sqlalchemy; print('[+] All core build & packaging dependencies verified.')"
|
||||
|
||||
- name: Compile Standalone Binaries (Linux & Windows)
|
||||
run: |
|
||||
|
||||
+11
-3
@@ -17,23 +17,31 @@ out/
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
*.nsh
|
||||
*.iss.tmp
|
||||
version.nsh
|
||||
version.iss
|
||||
|
||||
# Database files & state
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
client_state.json
|
||||
*.tmp
|
||||
|
||||
# Live configuration with generated private keys & tokens (samples are tracked)
|
||||
server_config.json
|
||||
client_config.json
|
||||
client_state.json
|
||||
local_collector_state.json
|
||||
certs/
|
||||
*.bak
|
||||
test_*.json
|
||||
test_*.db
|
||||
test_pipeline_master_ca/
|
||||
test_pipeline_certs/
|
||||
master_ca_certs/
|
||||
.env
|
||||
*.env
|
||||
runner/.env
|
||||
updates/
|
||||
|
||||
# IDE & OS files
|
||||
.vscode/
|
||||
|
||||
+1
-1
Submodule LOGAR-LICENSING updated: 7b43a4c82d...38a41cfbac
@@ -5,14 +5,14 @@
|
||||
# LOGAR: Fleet Log Aggregation & Centralized Provisioning System
|
||||
### *Secure Log Aggregation, PKI Fleet Licensing & Centralized Delivery for Autonomous AI Analysis*
|
||||
|
||||
[](RELEASE_NOTES.md)
|
||||
[](tests/)
|
||||
[](RELEASE_NOTES.md)
|
||||
[](tests/)
|
||||
[](https://gitea.eibl.tech/me0nline/LOGAR/wiki/PKI-and-Licensing)
|
||||
[](https://gitea.eibl.tech/me0nline/LOGAR/wiki/Branding-Guide)
|
||||
[](https://gitea.eibl.tech/me0nline/LOGAR/wiki/Admin-Guide)
|
||||
|
||||
**LOGAR** is an enterprise-grade log aggregation, normalization, and centralized provisioning system built for heterogeneous server fleets (Windows & Linux). It pairs ultra-lightweight, zero-state edge forwarders with a secure centralized server hub that enforces mutual TLS 1.3 (**mTLS**) authentication, dynamic PKI fleet licensing with cryptographic seat leasing, 12-hour temporal persistence tracking, and direct anomaly pass-through—delivering clean, structured, verified log telemetry directly to **Hermes AI** for autonomous deep reasoning, contextual root-cause analysis, and operational decision-making.
|
||||
**LOGAR** is an enterprise-grade log aggregation, normalization, and centralized provisioning system built for heterogeneous server fleets (Windows & Linux). It pairs ultra-lightweight, zero-state edge forwarders with a secure centralized server hub that enforces mutual TLS 1.3 (**mTLS**) authentication, dynamic PKI fleet licensing with cryptographic seat leasing, fleet auto-update package distribution, 12-hour temporal persistence tracking, and direct anomaly pass-through—delivering clean, structured, verified log telemetry directly to **Hermes AI** for autonomous deep reasoning, contextual root-cause analysis, and operational decision-making.
|
||||
|
||||
> 📚 **Comprehensive Technical Documentation**: Full architectural specifications, state machines, PKI X.509 OID definitions, branding assets, and complete REST API references are maintained on the **[LOGAR Gitea Wiki](https://gitea.eibl.tech/me0nline/LOGAR/wiki)**.
|
||||
> 📚 **WebUI Administrator Guide & Technical Wiki**: Complete WebUI console guides, storage switching, ingestion queue tuning, fleet package distribution, zero-knowledge encryption, and REST API references are available on the **[LOGAR Gitea Wiki](https://gitea.eibl.tech/me0nline/LOGAR/wiki)** and the **[WebUI Administrator Guide](https://gitea.eibl.tech/me0nline/LOGAR/wiki/Admin-Guide)**.
|
||||
|
||||
---
|
||||
|
||||
@@ -20,10 +20,14 @@
|
||||
|
||||
- **Zero-State Edge Forwarders**: Windows (`Win_Client`) and Linux (`Linux_Client`) agents maintain zero local databases, zero cache files, and minimal memory footprint.
|
||||
- **Mutual TLS 1.3 (`:9443`)**: Enforces bidirectional cryptographic identity verification (`ssl.CERT_REQUIRED`) for all streaming log frames.
|
||||
- **Hierarchical PKI & Seat Licensing**: Subordinate CAs automatically enroll and sync seat quotas with the Master Authority (`logar.eibl.tech`) over WebSockets.
|
||||
- **Hierarchical PKI & Seat Licensing**: Subordinate CAs automatically enroll and sync seat quotas with the Master Authority (`logar.eibl.tech`) over WebSockets. Active module capabilities (`enterprise_db`, `auto_update`) are introspected dynamically.
|
||||
- **Fleet Auto-Update & Binary Distribution**: Centralized local binary distribution point with SHA-256 integrity verification, automated client version negotiation, and background pull orchestrator.
|
||||
- **Multi-Engine Storage Backend**: Switch seamlessly between lightweight local SQLite (WAL mode) and external enterprise databases (PostgreSQL, MySQL, MariaDB) with automated schema migration.
|
||||
- **High-Throughput Ring Buffer**: In-memory ingestion queue with configurable batch flushing to eliminate disk I/O bottlenecks during peak traffic bursts.
|
||||
- **Zero-Knowledge Data Encryption**: End-to-end payload encryption at rest using server X.509 RSA-2048 / OpenPGP keys for external SQL databases.
|
||||
- **12-Hour Temporal Persistence**: Evaluates anomalies over a rolling 12-hour temporal window, tracking occurrence velocity and first/last seen metrics.
|
||||
- **Direct Hermes AI Provisioning (`:8443`)**: Exposes verified anomalous telemetry directly to Hermes AI via `/api/hermes/report` without heuristic bottlenecks.
|
||||
- **WebUI Dashboard**: Authenticated, 2-tab management console for live anomaly telemetry, client seat leasing, and Sub-CA lifecycle management.
|
||||
- **WebUI Dashboard**: Authenticated, clean management console for live anomaly telemetry, client seat leasing, modular licensing status, storage engine configuration, and Sub-CA lifecycle management.
|
||||
|
||||
---
|
||||
|
||||
@@ -149,16 +153,43 @@ LOGAR/
|
||||
|
||||
- **`POST /api/client/enroll`**: Forwarder auto-enrollment and proactive certificate renewal.
|
||||
- **`GET /api/clients`**: Returns active seat quota and registered edge forwarder telemetry *(Admin Auth)*.
|
||||
- **`GET /api/client/update/check`**: Evaluates client version and returns upgrade manifest with SHA-256 integrity digest.
|
||||
- **`GET /api/client/update/download`**: Delivers OS-specific standalone binary packages for forwarder upgrades.
|
||||
- **`GET /api/hermes/report`**: Structured JSON anomaly stream for Hermes AI reasoning.
|
||||
- **`POST /api/license/sync`**: Triggers immediate WebSocket sync with `logar.eibl.tech` *(Admin Auth)*.
|
||||
- **`GET /health`**: Server health, encryption fingerprint, and mTLS status.
|
||||
|
||||
---
|
||||
|
||||
## Enterprise Feature Capabilities
|
||||
|
||||
LOGAR uses cryptographic X.509 v3 custom extensions embedded inside activated `.lic` intermediate CA certificates issued by the Master Authority:
|
||||
|
||||
| Feature Flag | Capability Name | Description | Default Status |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| **`enterprise_db`** | **External DB & Ingestion Buffer** | Unlocks PostgreSQL/MySQL storage backends, high-throughput in-memory RAM buffering with batch flushing, and OpenPGP zero-knowledge at-rest data encryption. | Enabled by default |
|
||||
| **`auto_update`** | **Fleet Auto-Update Orchestration** | Enables central repository binary hosting, client version discovery, background update polling, and automated binary hot-upgrades. | Enabled by default |
|
||||
|
||||
---
|
||||
|
||||
## Fleet Auto-Update & Package Distribution
|
||||
|
||||
LOGAR operates a pull-based fleet package distribution architecture:
|
||||
|
||||
1. **Central Server Repository**: The Server Hub monitors upstream releases and hosts target client binaries (`LOGAR_Client_Setup.exe`, `Linux_Client.bin`) in the local repository (`/opt/logar-server/updates/` or `.\updates\`).
|
||||
2. **Version Negotiation**: Edge clients periodically contact `GET /api/client/update/check?os=<windows|linux>&version=<current>` to compare active versions against the Hub.
|
||||
3. **Integrity Verification**: Client packages are validated using streaming SHA-256 digests prior to installation or hot-swap.
|
||||
4. **Client CLI Flags**:
|
||||
- `--auto-update`: Enables automated background update polling and downloading.
|
||||
- `--update-check-interval <seconds>`: Configures update poll frequency (default: 3600s).
|
||||
- `--check-update-now`: Queries server for available updates and exits.
|
||||
|
||||
---
|
||||
|
||||
## Testing & Cross-Compilation
|
||||
|
||||
```bash
|
||||
# Run full unit test suite (42 tests)
|
||||
# Run full unit test suite (51 tests)
|
||||
python -m unittest discover -s tests -v
|
||||
|
||||
# Run end-to-end mTLS pipeline integration test
|
||||
@@ -174,10 +205,11 @@ python compilation/package_dist.py --target all
|
||||
|
||||
For in-depth operational guides, lifecycle state machines, and PKI specifications, consult the **[LOGAR Gitea Wiki](https://gitea.eibl.tech/me0nline/LOGAR/wiki)**:
|
||||
|
||||
* [Branding & Design System Guide](https://gitea.eibl.tech/me0nline/LOGAR/wiki/Branding-Guide)
|
||||
* [WebUI Administrator Guide](https://gitea.eibl.tech/me0nline/LOGAR/wiki/Admin-Guide)
|
||||
* [Architecture & Design Details](https://gitea.eibl.tech/me0nline/LOGAR/wiki/Architecture-and-Design)
|
||||
* [System State Machines & Subsystem Lifecycles](https://gitea.eibl.tech/me0nline/LOGAR/wiki/State-Machines-and-Lifecycles)
|
||||
* [PKI Architecture & Custom X.509 Seat OIDs](https://gitea.eibl.tech/me0nline/LOGAR/wiki/PKI-and-Licensing)
|
||||
* [Complete REST API Specification](https://gitea.eibl.tech/me0nline/LOGAR/wiki/API-Reference)
|
||||
* [Production Deployment & Service Management](https://gitea.eibl.tech/me0nline/LOGAR/wiki/Deployment-and-Installation)
|
||||
* [CI/CD Pipeline & Cross-Compilation Workflow](https://gitea.eibl.tech/me0nline/LOGAR/wiki/Testing-and-Releases)
|
||||
* [Branding & Design System Guide](https://gitea.eibl.tech/me0nline/LOGAR/wiki/Branding-Guide)
|
||||
|
||||
+46
-25
@@ -1,37 +1,58 @@
|
||||
# LOGAR Release Notes
|
||||
|
||||
## [v0.3.4] - 2026-09-27
|
||||
## [v0.3.5.2] - 2026-10-03
|
||||
|
||||
### 🔄 In-Place Installer Upgrades & Mutual Exclusion Guards
|
||||
- **Seamless In-Place Upgrades**: Windows (Inno Setup & NSIS) and Linux installers now detect existing installations automatically. In upgrade mode, installers stop background services, update core binaries and runtime in-place, and restart services without overwriting configuration files (`server_config.json`, `client_config.json`), database state (`logar_state.db`), state tracking (`client_state.json`), or mTLS certificates.
|
||||
- **Host Mutual Exclusion Validation**: Added pre-installation validation across all installers to prevent accidental co-installation of LOGAR Server and LOGAR Client on the same host, showing descriptive guidance to administrators.
|
||||
### 🚀 Fleet Hub Client Update Caching & Distribution
|
||||
- **Local Package Caching**: The Fleet Hub automatically caches edge client binaries and packages locally on startup and upon scheduled refresh, minimizing external upstream traffic.
|
||||
- **Edge Pull Mechanism**: Forwarder nodes retrieve validated update payloads directly from the local Fleet Hub.
|
||||
|
||||
### 🛡️ WebUI 2FA & Telemetry Stability
|
||||
- **Server Hub TOTP Two-Factor Authentication**: Implemented RFC 6238 TOTP 2FA for the LOGAR Server Hub web portal (`:8443`), featuring interactive QR code setup, secret generation, and secure session validation.
|
||||
- **Telemetry Stream UI Buffer Capping**: Limited the live verified telemetry stream view to the latest 50 events in the Server Hub dashboard for enhanced browser performance and UI stability.
|
||||
### 🛡️ Windows Defender Firewall Automation (Fixes Issue #1)
|
||||
- **Automated Port Exemptions**: Inno Setup (`installer_server.iss`) and NSIS (`installer_server.nsi`) server installers automatically configure inbound Windows Defender Firewall rules for WebUI (`TCP 8443`) and mTLS log ingestion (`TCP 9443`).
|
||||
- **Clean Uninstallation**: Uninstallers automatically remove the created firewall rules cleanly upon server decommissioning.
|
||||
|
||||
### 🔑 Licensing Authority Management & Registry Tooling
|
||||
- **Revocation, Reinstatement & License Deletion**: Enhanced the Master Licensing Authority (`logar.eibl.tech`) with instant license revocation, reinstatement of revoked licenses, and conditional permanent deletion of revoked records.
|
||||
- **Interactive Capability Explorer**: Added interactive implementation breakdown modals across core capabilities on the public project landing page.
|
||||
- **Automated Gitea Container Pruning**: Added `prune_images.py` utility to maintain clean container registries by keeping only the latest active image versions.
|
||||
### 📊 Forwarder Liveness, Silent Server Detection & System Telemetry (Fixes Issue #2)
|
||||
- **Silent Node & Crash Detection**: Resolved the "Silent Server Down Bug" by tracking forwarder heartbeats (`last_seen`). Hermes reporting (`GET /api/hermes/report`) automatically flags edge nodes that have gone silent for >15 minutes as `CRITICAL` `NODE_OFFLINE` alerts.
|
||||
- **Troubleshooting System Telemetry**: Cross-platform system telemetry metrics (CPU utilization %, Memory load %, and Free Disk space %) are collected and attached to client heartbeats and Hermes anomaly records without external dependencies.
|
||||
- **Benign Service Noise Suppression**: Filtered routine repetitive Windows and Linux background service lifecycle events (e.g., `sppsvc`, `edgeupdate`, `trustedinstaller`, `wuauserv`, `backgroundTaskHost`) from false-positive anomaly alert fatigue.
|
||||
|
||||
### 🔑 Dynamic Enrollment Key Rotation & WebUI Key Management
|
||||
- **Automatic Key Invalidation**: Clicking "Download Client Config" (`GET /api/client/export-profile`) automatically generates a fresh enrollment secret and invalidates previous secrets, preventing reuse of leaked keys.
|
||||
- **WebUI Reset Shared Key**: Added "🔄 Reset Shared Key" controls in the WebUI header, Fleet tab, and Export Modal, backed by `POST /api/server/reset-enrollment-secret` to rotate keys and update quick-deploy command snippets on demand.
|
||||
|
||||
### 🔒 Security Hardening & Modular Architecture
|
||||
- **Decoupled CI & Test Execution**: Clean modularization across server, client, and authority services with fully autonomous unit and integration test suites.
|
||||
- **API Robustness & Validation**: Added cryptographic CSR validation endpoints and request sanitization.
|
||||
|
||||
---
|
||||
|
||||
## [v0.3.3] - 2026-09-27
|
||||
## [v0.3.5.1] - 2026-09-30
|
||||
|
||||
### 🎨 Brand Identity, Centralized Asset Library & Design System
|
||||
- **Centralized `/assets` Repository**: Added official vector graphics (`logo.svg`, `logo_transparent.svg`, `logo_dark.svg`, `logo_banner.svg`), high-resolution master PNGs (16px to 512px), multi-resolution Windows ICOs (`logo.ico`, `favicon.ico`), and installer dialog bitmaps (`wizard_header.bmp`, `wizard_small.bmp`, `wizard_sidebar.bmp`).
|
||||
- **Official Wiki Branding Guide**: Published comprehensive visual identity standards on the Gitea Wiki at `https://gitea.eibl.tech/me0nline/LOGAR/wiki/Branding-Guide` detailing logo anatomy, color palettes (Cyber Cyan `#00E5FF`, Emerald `#00FFA3`, Dark Navy `#0E1A2E`), typography, clear-space rules, dos and don'ts, and strict non-SIEM positioning.
|
||||
- **Unified Interface & Tab Branding**: Implemented transparent vector logos on all web navigation bars and browser tabs across both the LOGAR Server Hub (`:8443`) and Master Licensing Authority (`logar.eibl.tech`).
|
||||
- **Clean Installer Bitmaps**: Regenerated 24-bit RGB header bitmaps with seamless white backgrounds (`#FFFFFF`) matching native Windows Inno Setup wizard dialogs.
|
||||
- **Automated Asset Pipeline**: Enhanced `compilation/generate_branding_assets.py` to deterministically compile and distribute assets across main repo, compilation subdirectories, and the licensing submodule without third-party graphics dependencies.
|
||||
### 🗄️ Multi-Engine Storage Architecture
|
||||
- **Dynamic Backend Switching**: LOGAR Server Hub supports runtime switching between lightweight Local SQLite (WAL Mode) and Enterprise SQL servers (PostgreSQL and MySQL / MariaDB) via SQLAlchemy.
|
||||
- **Automated Schema Provisioning**: Database switching automatically verifies connectivity, provisions normalized tables (`telemetry_events`, `client_seats`, `license_config`, `admin_users`), and generates optimized query indices.
|
||||
- **Config Isolation**: Runtime backend parameters are stored persistently in `server_config.json` under `db_engine` and safely isolated from ephemeral test runs.
|
||||
|
||||
---
|
||||
### ⚡ Ingestion Ring Buffer & Burst Protection
|
||||
- **In-Memory Queue Buffering**: Built-in ring buffer with an asynchronous background flusher decoupled from mTLS socket I/O.
|
||||
- **Cache Mode**: High-velocity log spikes are absorbed in memory without blocking incoming mTLS 1.3 frame streams.
|
||||
- **Configurable Batching**: Configurable flush interval (`buffer_flush_interval_sec`, default 2.0s) and batch size (`buffer_batch_size`, default 100).
|
||||
- **Graceful Shutdown**: Unflushed queue items are flushed and committed cleanly during server shutdown.
|
||||
- **Opt-In Security Policy**: Buffering remains an explicit administrative choice and is not auto-enabled upon license activation.
|
||||
|
||||
## [v0.3.2] - 2026-09-27
|
||||
### 🔐 Zero-Knowledge OpenPGP Data Encryption at Rest
|
||||
- **Public-Key Envelope Encryption**: External SQL payloads (`event_data`) can be encrypted at rest using the server's X.509 RSA-2048 / OpenPGP public key with AES-256-GCM symmetric session keys.
|
||||
- **Zero-Knowledge on Database Host**: External DBAs or compromised database hosts cannot read plaintext telemetry without the server's private key.
|
||||
- **Opt-In Security Policy**: Payload encryption is strictly opt-in and configurable per compliance requirements.
|
||||
|
||||
### 🪟 Windows Installer & Web Portal Enhancements
|
||||
- **Project Showcase Landing Page**: Deployed high-impact public homepage at `GET /` on the licensing authority with architecture diagrams, quickstart commands, and direct navigation to Gitea, Wiki, and the Licensing Portal (`/licensing`).
|
||||
- **Standalone Windows Binaries & Python Runtime**: Resolved issues with standalone Windows executable execution and bundled runtime paths.
|
||||
- **Data Directory Selection & Permissions**: Added custom data directory selection during setup and fixed permissions for background Windows services.
|
||||
- **Service Lifecycle & Clean Uninstallation**: Resolved service startup issues and ensured complete cleanup of state files upon uninstallation.
|
||||
### 📜 Dynamic Licensing Introspection & Clean Dashboard WebUI
|
||||
- **Modular Capability Introspection**: PKI X.509 extension OID `1.3.6.1.4.1.58888.1.3` and persistent license records are dynamically parsed to display active modules (`enterprise_db`, etc.) in a dedicated card on the **📜 Sub-CA & Licensing** tab.
|
||||
- **Distraction-Free Tab Navigation**: Cleaned all dashboard tab headers to remove redundant item counts and visual badges for a streamlined administrative interface.
|
||||
- **Graceful Feature Locking**: Unlicensed enterprise controls remain visible but gracefully disabled in the WebUI and rejected by backend API validation.
|
||||
|
||||
### 🛠️ Dynamic Installer Context Versioning & Streamlined CI/CD
|
||||
- **Context-Aware Installer Builds**: NSIS and Inno Setup compiler scripts dynamically derive version information directly from build context/tags (`v0.3.5.1`) without hardcoded version strings.
|
||||
- **Pre-Integrated Builder Runner**: Streamlined CI/CD workflows to rely directly on pre-installed container tools and libraries without runtime `apt`/`pip` overhead.
|
||||
|
||||
### 📖 WebUI Administrator Guide & 51 Automated Tests
|
||||
- **WebUI Administrator Guide**: Published comprehensive WebUI management guide on the [Gitea Wiki](https://gitea.eibl.tech/me0nline/LOGAR/wiki/Admin-Guide) covering console access, telemetry filtering, seat management, licensing, database switching, queue tuning, zero-knowledge crypto, Hermes AI, and 2FA.
|
||||
- **Expanded Test Suite**: 51 automated unit and integration tests passing with 100% test coverage.
|
||||
|
||||
@@ -14,9 +14,12 @@ import shutil
|
||||
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
ROOT_ASSETS_DIR = os.path.join(ROOT_DIR, "assets")
|
||||
COMPILATION_ASSETS_DIR = os.path.join(ROOT_DIR, "compilation", "assets")
|
||||
LICENSING_ASSETS_DIR = os.path.join(ROOT_DIR, "LOGAR-LICENSING", "assets")
|
||||
target_asset_dirs = [ROOT_ASSETS_DIR, COMPILATION_ASSETS_DIR]
|
||||
licensing_dir = os.path.join(ROOT_DIR, "LOGAR-LICENSING")
|
||||
if os.path.exists(licensing_dir):
|
||||
target_asset_dirs.append(os.path.join(licensing_dir, "assets"))
|
||||
|
||||
for d in (ROOT_ASSETS_DIR, COMPILATION_ASSETS_DIR, LICENSING_ASSETS_DIR):
|
||||
for d in target_asset_dirs:
|
||||
os.makedirs(d, exist_ok=True)
|
||||
|
||||
|
||||
@@ -419,7 +422,7 @@ def main():
|
||||
swatch_svgs[name] = f'<svg width="18" height="18" viewBox="0 0 18 18" xmlns="http://www.w3.org/2000/svg"><rect x="1" y="1" width="16" height="16" rx="4" fill="{hex_code}" stroke="{border}" stroke-width="1.5"/></svg>'
|
||||
|
||||
# 6. Write to target directories
|
||||
for target_dir in (ROOT_ASSETS_DIR, COMPILATION_ASSETS_DIR, LICENSING_ASSETS_DIR):
|
||||
for target_dir in target_asset_dirs:
|
||||
swatch_dir = os.path.join(target_dir, "swatches")
|
||||
os.makedirs(swatch_dir, exist_ok=True)
|
||||
|
||||
|
||||
@@ -1,7 +1,15 @@
|
||||
#ifexist "version.iss"
|
||||
#include "version.iss"
|
||||
#endif
|
||||
|
||||
#ifndef AppVersion
|
||||
#define AppVersion "0.3.5.1"
|
||||
#endif
|
||||
|
||||
[Setup]
|
||||
AppId={{9C4A6F1D-8975-476E-8512-4217DE029A88}}
|
||||
AppName=LOGAR Client
|
||||
AppVersion=0.3.4
|
||||
AppVersion={#AppVersion}
|
||||
AppPublisher=LOGAR
|
||||
AppPublisherURL=https://logar.eibl.tech
|
||||
AppSupportURL=https://logar.eibl.tech
|
||||
@@ -19,9 +27,9 @@ WizardImageFile=assets\wizard_sidebar.bmp
|
||||
WizardSmallImageFile=assets\wizard_small.bmp
|
||||
VersionInfoCompany=LOGAR
|
||||
VersionInfoDescription=LOGAR Windows Edge Log Forwarder Setup
|
||||
VersionInfoVersion=0.3.4.0
|
||||
VersionInfoVersion={#AppVersion}
|
||||
VersionInfoProductName=LOGAR Client
|
||||
VersionInfoProductVersion=0.3.4
|
||||
VersionInfoProductVersion={#AppVersion}
|
||||
|
||||
[Files]
|
||||
Source: "..\dist\Win_Client.exe"; DestDir: "{app}"; Flags: ignoreversion
|
||||
|
||||
@@ -7,15 +7,24 @@ OutFile "../dist/LOGAR-Client-Setup.exe"
|
||||
InstallDir "$PROGRAMFILES64\LOGAR"
|
||||
RequestExecutionLevel admin
|
||||
|
||||
BrandingText "LOGAR Centralized Log Aggregator & Forwarder"
|
||||
BrandingText "LOGAR Centralized Log Aggregator && Forwarder"
|
||||
|
||||
VIProductVersion "0.3.4.0"
|
||||
!include /NONFATAL "version.nsh"
|
||||
|
||||
!ifndef APP_VERSION
|
||||
!define APP_VERSION "0.0.0.0"
|
||||
!endif
|
||||
!ifndef DISPLAY_VERSION
|
||||
!define DISPLAY_VERSION "${APP_VERSION}"
|
||||
!endif
|
||||
|
||||
VIProductVersion "${APP_VERSION}"
|
||||
VIAddVersionKey "ProductName" "LOGAR Client"
|
||||
VIAddVersionKey "CompanyName" "LOGAR"
|
||||
VIAddVersionKey "LegalCopyright" "© 2026 LOGAR"
|
||||
VIAddVersionKey "FileDescription" "LOGAR Windows Edge Log Forwarder Setup"
|
||||
VIAddVersionKey "FileVersion" "0.3.4.0"
|
||||
VIAddVersionKey "ProductVersion" "0.3.4.0"
|
||||
VIAddVersionKey "FileVersion" "${APP_VERSION}"
|
||||
VIAddVersionKey "ProductVersion" "${DISPLAY_VERSION}"
|
||||
|
||||
# --- Visual Graphics & Branding ---
|
||||
!define MUI_ICON "assets/logo.ico"
|
||||
@@ -127,13 +136,13 @@ Function ConfigPageCreate
|
||||
|
||||
!insertmacro MUI_HEADER_TEXT "Import Client Configuration (Optional)" "Choose a client_config.json profile exported from LOGAR Server."
|
||||
|
||||
${NSD_CreateLabel} 0 0 100% 30u "If you have exported a client_config.json profile bundle from your LOGAR Server dashboard, select it below to configure the forwarder immediately.$\r$\n$\r$\nLeave this field blank to configure later."
|
||||
${NSD_CreateLabel} 0 0 100% 48u "If you have exported a client_config.json profile bundle from your LOGAR Server dashboard, select it below to configure the forwarder immediately.$\r$\n$\r$\nLeave this field blank to configure later."
|
||||
Pop $0
|
||||
|
||||
${NSD_CreateText} 0 38u 75% 12u "$CFGFILE"
|
||||
${NSD_CreateText} 0 54u 75% 14u "$CFGFILE"
|
||||
Pop $CfgFileInput
|
||||
|
||||
${NSD_CreateButton} 78% 37u 22% 14u "Browse..."
|
||||
${NSD_CreateButton} 78% 53u 22% 16u "Browse..."
|
||||
Pop $CfgBrowseBtn
|
||||
${NSD_OnClick} $CfgBrowseBtn OnBrowseCfg
|
||||
|
||||
|
||||
@@ -1,7 +1,15 @@
|
||||
#ifexist "version.iss"
|
||||
#include "version.iss"
|
||||
#endif
|
||||
|
||||
#ifndef AppVersion
|
||||
#define AppVersion "0.3.5.1"
|
||||
#endif
|
||||
|
||||
[Setup]
|
||||
AppId={{4F3D675E-1E4E-43B6-8C84-B47DE290BE4A}}
|
||||
AppName=LOGAR Server
|
||||
AppVersion=0.3.4
|
||||
AppVersion={#AppVersion}
|
||||
AppPublisher=LOGAR
|
||||
AppPublisherURL=https://logar.eibl.tech
|
||||
AppSupportURL=https://logar.eibl.tech
|
||||
@@ -19,9 +27,9 @@ WizardImageFile=assets\wizard_sidebar.bmp
|
||||
WizardSmallImageFile=assets\wizard_small.bmp
|
||||
VersionInfoCompany=LOGAR
|
||||
VersionInfoDescription=LOGAR Cloud Hub & TCP Ingestion Server Setup
|
||||
VersionInfoVersion=0.3.4.0
|
||||
VersionInfoVersion={#AppVersion}
|
||||
VersionInfoProductName=LOGAR Server
|
||||
VersionInfoProductVersion=0.3.4
|
||||
VersionInfoProductVersion={#AppVersion}
|
||||
|
||||
[Files]
|
||||
Source: "..\dist\Server.exe"; DestDir: "{app}"; Flags: ignoreversion
|
||||
@@ -37,9 +45,13 @@ Filename: "{app}\nssm.exe"; Parameters: "set LOGAR_Server AppDirectory ""{app}""
|
||||
Filename: "{app}\nssm.exe"; Parameters: "set LOGAR_Server AppStdout ""{code:GetDataDir}\server.log"""; Flags: runhidden
|
||||
Filename: "{app}\nssm.exe"; Parameters: "set LOGAR_Server AppStderr ""{code:GetDataDir}\server_err.log"""; Flags: runhidden
|
||||
Filename: "{app}\nssm.exe"; Parameters: "set LOGAR_Server AppParameters ""--config """"{code:GetDataDir}\server_config.json"""""""; Flags: runhidden
|
||||
Filename: "netsh.exe"; Parameters: "advfirewall firewall add rule name=""LOGAR Server WebUI (8443)"" dir=in action=allow protocol=TCP localport=8443"; Flags: runhidden
|
||||
Filename: "netsh.exe"; Parameters: "advfirewall firewall add rule name=""LOGAR Server Ingestion (9443)"" dir=in action=allow protocol=TCP localport=9443"; Flags: runhidden
|
||||
Filename: "{app}\nssm.exe"; Parameters: "start LOGAR_Server"; Flags: runhidden
|
||||
|
||||
[UninstallRun]
|
||||
Filename: "netsh.exe"; Parameters: "advfirewall firewall delete rule name=""LOGAR Server WebUI (8443)"""; Flags: runhidden
|
||||
Filename: "netsh.exe"; Parameters: "advfirewall firewall delete rule name=""LOGAR Server Ingestion (9443)"""; Flags: runhidden
|
||||
Filename: "{app}\nssm.exe"; Parameters: "stop LOGAR_Server"; Flags: runhidden
|
||||
Filename: "{app}\nssm.exe"; Parameters: "remove LOGAR_Server confirm"; Flags: runhidden
|
||||
|
||||
|
||||
@@ -6,15 +6,24 @@ OutFile "../dist/LOGAR-Server-Setup.exe"
|
||||
InstallDir "$PROGRAMFILES64\LOGAR-Server"
|
||||
RequestExecutionLevel admin
|
||||
|
||||
BrandingText "LOGAR Centralized Log Aggregator & Ingestion Hub"
|
||||
BrandingText "LOGAR Centralized Log Aggregator && Ingestion Hub"
|
||||
|
||||
VIProductVersion "0.3.4.0"
|
||||
!include /NONFATAL "version.nsh"
|
||||
|
||||
!ifndef APP_VERSION
|
||||
!define APP_VERSION "0.0.0.0"
|
||||
!endif
|
||||
!ifndef DISPLAY_VERSION
|
||||
!define DISPLAY_VERSION "${APP_VERSION}"
|
||||
!endif
|
||||
|
||||
VIProductVersion "${APP_VERSION}"
|
||||
VIAddVersionKey "ProductName" "LOGAR Server"
|
||||
VIAddVersionKey "CompanyName" "LOGAR"
|
||||
VIAddVersionKey "LegalCopyright" "© 2026 LOGAR"
|
||||
VIAddVersionKey "FileDescription" "LOGAR Cloud Hub & TCP Ingestion Server Setup"
|
||||
VIAddVersionKey "FileVersion" "0.3.4.0"
|
||||
VIAddVersionKey "ProductVersion" "0.3.4.0"
|
||||
VIAddVersionKey "FileVersion" "${APP_VERSION}"
|
||||
VIAddVersionKey "ProductVersion" "${DISPLAY_VERSION}"
|
||||
|
||||
# --- Visual Graphics & Branding ---
|
||||
!define MUI_ICON "assets/logo.ico"
|
||||
@@ -123,6 +132,10 @@ Section "Install"
|
||||
CreateDirectory "$DATADIR"
|
||||
ExecWait 'icacls "$DATADIR" /grant "*S-1-5-32-545:(OI)(CI)M" /T /Q'
|
||||
|
||||
# Configure Windows Defender Firewall inbound rules for WebUI (8443) and Ingestion (9443)
|
||||
ExecWait 'netsh advfirewall firewall add rule name="LOGAR Server WebUI (8443)" dir=in action=allow protocol=TCP localport=8443'
|
||||
ExecWait 'netsh advfirewall firewall add rule name="LOGAR Server Ingestion (9443)" dir=in action=allow protocol=TCP localport=9443'
|
||||
|
||||
# Install and configure service via NSSM
|
||||
ExecWait '"$INSTDIR\nssm.exe" install LOGAR_Server "$INSTDIR\Server.exe"'
|
||||
ExecWait '"$INSTDIR\nssm.exe" set LOGAR_Server AppDirectory "$INSTDIR"'
|
||||
@@ -143,7 +156,9 @@ Section "Uninstall"
|
||||
StrCpy $DATADIR "$APPDATA\LOGAR-Server"
|
||||
${EndIf}
|
||||
|
||||
# 1. Stop and remove service
|
||||
# 1. Stop and remove service & clean firewall rules
|
||||
ExecWait 'netsh advfirewall firewall delete rule name="LOGAR Server WebUI (8443)"'
|
||||
ExecWait 'netsh advfirewall firewall delete rule name="LOGAR Server Ingestion (9443)"'
|
||||
ExecWait 'net stop LOGAR_Server'
|
||||
ExecWait 'sc stop LOGAR_Server'
|
||||
${If} ${FileExists} "$INSTDIR\nssm.exe"
|
||||
|
||||
+179
-21
@@ -60,11 +60,17 @@ def write_checksum_file(filename, digest, binary_filename):
|
||||
|
||||
|
||||
def build_pyinstaller_binary(script_path, binary_name):
|
||||
print(f"[*] Compiling {binary_name} with PyInstaller...")
|
||||
print(f"[*] Compiling {binary_name} with PyInstaller (Bytecode optimized & stripped)...")
|
||||
sep = ";" if platform.system() == "Windows" else ":"
|
||||
cmd = [
|
||||
sys.executable, "-m", "PyInstaller",
|
||||
"--onefile",
|
||||
"--clean",
|
||||
"--strip",
|
||||
"--optimize", "2",
|
||||
"--paths", ROOT_DIR,
|
||||
"--add-data", f"{SRC_DIR}{sep}src",
|
||||
"--collect-all", "src",
|
||||
"--distpath", DIST_DIR,
|
||||
"--workpath", os.path.join(BUILD_TEMP, f"work_{binary_name}"),
|
||||
"--specpath", os.path.join(BUILD_TEMP, f"spec_{binary_name}"),
|
||||
@@ -82,10 +88,14 @@ def build_linux_zipapp_fallback():
|
||||
print("[*] Packaging Linux standalone zipapp fallback binaries...")
|
||||
# Linux Client zipapp
|
||||
app_dir = os.path.join(BUILD_TEMP, "linux_app")
|
||||
if os.path.exists(app_dir):
|
||||
shutil.rmtree(app_dir)
|
||||
os.makedirs(app_dir, exist_ok=True)
|
||||
shutil.copytree(SRC_DIR, os.path.join(app_dir, "src"), ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
|
||||
for f in os.listdir(SRC_DIR):
|
||||
if f.endswith(".py"):
|
||||
shutil.copy(os.path.join(SRC_DIR, f), os.path.join(app_dir, f))
|
||||
src_f = os.path.join(SRC_DIR, f)
|
||||
if os.path.isfile(src_f) and f.endswith(".py"):
|
||||
shutil.copy(src_f, os.path.join(app_dir, f))
|
||||
client_out = os.path.join(DIST_DIR, "Linux_Client.bin")
|
||||
zipapp.create_archive(
|
||||
source=app_dir,
|
||||
@@ -95,10 +105,14 @@ def build_linux_zipapp_fallback():
|
||||
)
|
||||
# Server zipapp
|
||||
srv_dir = os.path.join(BUILD_TEMP, "linux_srv")
|
||||
if os.path.exists(srv_dir):
|
||||
shutil.rmtree(srv_dir)
|
||||
os.makedirs(srv_dir, exist_ok=True)
|
||||
shutil.copytree(SRC_DIR, os.path.join(srv_dir, "src"), ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
|
||||
for f in os.listdir(SRC_DIR):
|
||||
if f.endswith(".py"):
|
||||
shutil.copy(os.path.join(SRC_DIR, f), os.path.join(srv_dir, f))
|
||||
src_f = os.path.join(SRC_DIR, f)
|
||||
if os.path.isfile(src_f) and f.endswith(".py"):
|
||||
shutil.copy(src_f, os.path.join(srv_dir, f))
|
||||
server_out = os.path.join(DIST_DIR, "Server.bin")
|
||||
zipapp.create_archive(
|
||||
source=srv_dir,
|
||||
@@ -195,11 +209,15 @@ def create_windows_exe_zipapp(script_name, exe_name):
|
||||
the standard Python Windows 64-bit PE launcher stub (t64.exe) to a compiled zipapp archive.
|
||||
"""
|
||||
app_temp_dir = os.path.join(BUILD_TEMP, f"win_app_{exe_name}")
|
||||
if os.path.exists(app_temp_dir):
|
||||
shutil.rmtree(app_temp_dir)
|
||||
os.makedirs(app_temp_dir, exist_ok=True)
|
||||
|
||||
shutil.copytree(SRC_DIR, os.path.join(app_temp_dir, "src"), ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
|
||||
for f in os.listdir(SRC_DIR):
|
||||
if f.endswith(".py"):
|
||||
shutil.copy(os.path.join(SRC_DIR, f), os.path.join(app_temp_dir, f))
|
||||
src_f = os.path.join(SRC_DIR, f)
|
||||
if os.path.isfile(src_f) and f.endswith(".py"):
|
||||
shutil.copy(src_f, os.path.join(app_temp_dir, f))
|
||||
|
||||
module_name = os.path.splitext(script_name)[0]
|
||||
zipapp_temp = os.path.join(BUILD_TEMP, f"{exe_name}.zip")
|
||||
@@ -290,6 +308,128 @@ def ensure_libpython():
|
||||
pass
|
||||
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def get_context_version():
|
||||
"""
|
||||
Extracts the current release version from the build context:
|
||||
1. Environment variables (APP_VERSION, LOGAR_VERSION, GITEA_REF_NAME, GITHUB_REF_NAME, GITHUB_REF)
|
||||
2. Git tag via `git describe --tags --exact-match` or `git describe --tags`
|
||||
3. RELEASE_NOTES.md top header
|
||||
4. README.md version badge
|
||||
Returns tuple: (display_version, four_part_version)
|
||||
e.g. ('0.3.5.1', '0.3.5.1') or ('0.3.5', '0.3.5.0')
|
||||
"""
|
||||
raw_version = ""
|
||||
|
||||
# 1. Environment variables
|
||||
for env_var in ["APP_VERSION", "LOGAR_VERSION", "GITEA_REF_NAME", "GITHUB_REF_NAME", "GITHUB_REF"]:
|
||||
val = os.environ.get(env_var, "").strip()
|
||||
if val:
|
||||
if val.startswith("refs/tags/"):
|
||||
val = val[len("refs/tags/"):]
|
||||
if val.startswith("v") and len(val) > 1 and val[1].isdigit():
|
||||
val = val[1:]
|
||||
if any(c.isdigit() for c in val):
|
||||
raw_version = val
|
||||
break
|
||||
|
||||
# 2. Git command
|
||||
if not raw_version:
|
||||
try:
|
||||
res = subprocess.run(["git", "describe", "--tags", "--exact-match"], cwd=ROOT_DIR, capture_output=True, text=True, check=False)
|
||||
if res.returncode == 0 and res.stdout.strip():
|
||||
tag = res.stdout.strip()
|
||||
if tag.startswith("v") and len(tag) > 1 and tag[1].isdigit():
|
||||
tag = tag[1:]
|
||||
raw_version = tag
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not raw_version:
|
||||
try:
|
||||
res = subprocess.run(["git", "describe", "--tags"], cwd=ROOT_DIR, capture_output=True, text=True, check=False)
|
||||
if res.returncode == 0 and res.stdout.strip():
|
||||
tag = res.stdout.strip().split("-")[0]
|
||||
if tag.startswith("v") and len(tag) > 1 and tag[1].isdigit():
|
||||
tag = tag[1:]
|
||||
raw_version = tag
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 3. RELEASE_NOTES.md
|
||||
if not raw_version:
|
||||
rel_notes = os.path.join(ROOT_DIR, "RELEASE_NOTES.md")
|
||||
if os.path.exists(rel_notes):
|
||||
try:
|
||||
with open(rel_notes, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
m = re.search(r"##\s*\[v?([0-9.]+)\]", content)
|
||||
if m:
|
||||
raw_version = m.group(1).strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 4. README.md
|
||||
if not raw_version:
|
||||
readme = os.path.join(ROOT_DIR, "README.md")
|
||||
if os.path.exists(readme):
|
||||
try:
|
||||
with open(readme, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
m = re.search(r"version-v?([0-9.]+)-blue", content)
|
||||
if m:
|
||||
raw_version = m.group(1).strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback
|
||||
if not raw_version:
|
||||
raw_version = "0.3.5.1"
|
||||
|
||||
# Clean display version (strip non-standard characters, keep digits and dots)
|
||||
m_clean = re.search(r"([0-9]+(?:\.[0-9]+)*)", raw_version)
|
||||
display_version = m_clean.group(1) if m_clean else raw_version
|
||||
|
||||
# Format 4-part numeric version for NSIS VIProductVersion (X.X.X.X)
|
||||
parts = display_version.split(".")
|
||||
while len(parts) < 4:
|
||||
parts.append("0")
|
||||
four_part_version = ".".join(parts[:4])
|
||||
|
||||
return display_version, four_part_version
|
||||
|
||||
|
||||
def write_version_headers(display_version, four_part_version):
|
||||
"""Writes dynamic version header files for NSIS and Inno Setup compilers."""
|
||||
compilation_dir = os.path.join(ROOT_DIR, "compilation")
|
||||
os.makedirs(compilation_dir, exist_ok=True)
|
||||
|
||||
nsh_file = os.path.join(compilation_dir, "version.nsh")
|
||||
with open(nsh_file, "w", encoding="utf-8") as f:
|
||||
f.write(
|
||||
f"; Auto-generated version header from build context\n"
|
||||
f"!ifndef APP_VERSION\n"
|
||||
f' !define APP_VERSION "{four_part_version}"\n'
|
||||
f"!endif\n"
|
||||
f"!ifndef DISPLAY_VERSION\n"
|
||||
f' !define DISPLAY_VERSION "{display_version}"\n'
|
||||
f"!endif\n"
|
||||
)
|
||||
|
||||
iss_file = os.path.join(compilation_dir, "version.iss")
|
||||
with open(iss_file, "w", encoding="utf-8") as f:
|
||||
f.write(
|
||||
f"; Auto-generated version header from build context\n"
|
||||
f"#ifndef AppVersion\n"
|
||||
f' #define AppVersion "{display_version}"\n'
|
||||
f"#endif\n"
|
||||
)
|
||||
|
||||
print(f"[+] Dynamic context version resolved: {display_version} (NSIS VIProductVersion: {four_part_version})")
|
||||
|
||||
|
||||
def compile_nsis_installers():
|
||||
"""Compiles NSIS installers if makensis is available (native Linux/Windows)."""
|
||||
makensis_bin = ensure_makensis()
|
||||
@@ -297,20 +437,29 @@ def compile_nsis_installers():
|
||||
print("[!] Note: NSIS compiler (makensis) not found in PATH or standard directories.")
|
||||
return False
|
||||
|
||||
makensis_cmd = [makensis_bin]
|
||||
display_ver, vi_ver = get_context_version()
|
||||
write_version_headers(display_ver, vi_ver)
|
||||
|
||||
compilation_dir = os.path.join(ROOT_DIR, "compilation")
|
||||
success = True
|
||||
makensis_cmd = [
|
||||
makensis_bin,
|
||||
f"-DAPP_VERSION={vi_ver}",
|
||||
f"-DDISPLAY_VERSION={display_ver}"
|
||||
]
|
||||
failed_scripts = []
|
||||
for nsi in ["installer_client.nsi", "installer_server.nsi"]:
|
||||
nsi_path = os.path.join(compilation_dir, nsi)
|
||||
if os.path.exists(nsi_path):
|
||||
print(f"[*] Compiling NSIS script: {nsi} via {makensis_cmd[0]}...")
|
||||
print(f"[*] Compiling NSIS script: {nsi} (v{display_ver}) via {makensis_cmd[0]}...")
|
||||
try:
|
||||
subprocess.run(makensis_cmd + [nsi_path], cwd=compilation_dir, check=True)
|
||||
print(f"[+] Successfully compiled NSIS installer for {nsi}")
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Failed to compile {nsi} with {makensis_cmd}: {e}")
|
||||
success = False
|
||||
return success
|
||||
print(f"[!] Error: Failed to compile {nsi} with {makensis_cmd}: {e}")
|
||||
failed_scripts.append(nsi)
|
||||
if failed_scripts:
|
||||
raise RuntimeError(f"NSIS compilation failed for: {', '.join(failed_scripts)}")
|
||||
return True
|
||||
|
||||
|
||||
def compile_inno_setup_installers():
|
||||
@@ -338,20 +487,29 @@ def compile_inno_setup_installers():
|
||||
print("[!] Note: Inno Setup compiler (iscc / innosetup) not available on host. Skipping installer .exe creation.")
|
||||
return False
|
||||
|
||||
display_ver, vi_ver = get_context_version()
|
||||
write_version_headers(display_ver, vi_ver)
|
||||
|
||||
failed_scripts = []
|
||||
for iss in ["installer_client.iss", "installer_server.iss"]:
|
||||
iss_path = os.path.join(ROOT_DIR, "compilation", iss)
|
||||
if os.path.exists(iss_path):
|
||||
print(f"[*] Compiling Inno Setup script: {iss}...")
|
||||
print(f"[*] Compiling Inno Setup script: {iss} (v{display_ver})...")
|
||||
try:
|
||||
subprocess.run(iscc_cmd + [iss_path], check=True)
|
||||
subprocess.run(iscc_cmd + [f"/DAppVersion={display_ver}", iss_path], check=True)
|
||||
print(f"[+] Successfully compiled {iss}")
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Failed to compile {iss} with {iscc_cmd}: {e}")
|
||||
print(f"[!] Error: Failed to compile {iss} with {iscc_cmd}: {e}")
|
||||
failed_scripts.append(iss)
|
||||
if failed_scripts:
|
||||
raise RuntimeError(f"Inno Setup compilation failed for: {', '.join(failed_scripts)}")
|
||||
return True
|
||||
|
||||
|
||||
def compile_windows_installers():
|
||||
if not compile_nsis_installers():
|
||||
if ensure_makensis():
|
||||
compile_nsis_installers()
|
||||
else:
|
||||
compile_inno_setup_installers()
|
||||
|
||||
|
||||
@@ -406,14 +564,14 @@ def build_windows():
|
||||
write_checksum_file("win_agent_sha256sum", client_hash, "Win_Client.exe")
|
||||
sums.append(f"{client_hash} Win_Client.exe")
|
||||
else:
|
||||
print(f"[!] Warning: Expected {client_bin} was not found.")
|
||||
raise RuntimeError(f"Expected Windows client binary was not produced: {client_bin}")
|
||||
|
||||
if os.path.exists(server_bin):
|
||||
server_hash = calculate_sha256(server_bin)
|
||||
write_checksum_file("win_server_sha256sum", server_hash, "Server.exe")
|
||||
sums.append(f"{server_hash} Server.exe")
|
||||
else:
|
||||
print(f"[!] Warning: Expected {server_bin} was not found.")
|
||||
raise RuntimeError(f"Expected Windows server binary was not produced: {server_bin}")
|
||||
|
||||
# Inno Setup output checksums if present
|
||||
for inst in ["LOGAR-Client-Setup.exe", "LOGAR-Server-Setup.exe", "LOGAR-Windows-Client.zip", "LOGAR-Windows-Server.zip"]:
|
||||
@@ -500,14 +658,14 @@ def build_linux():
|
||||
write_checksum_file("linux_agent_sha256sum", client_hash, "Linux_Client.bin")
|
||||
sums.append(f"{client_hash} Linux_Client.bin")
|
||||
else:
|
||||
print(f"[!] Warning: Expected {client_bin} was not found.")
|
||||
raise RuntimeError(f"Expected Linux client binary was not produced: {client_bin}")
|
||||
|
||||
if os.path.exists(server_bin):
|
||||
server_hash = calculate_sha256(server_bin)
|
||||
write_checksum_file("linux_server_sha256sum", server_hash, "Server.bin")
|
||||
sums.append(f"{server_hash} Server.bin")
|
||||
else:
|
||||
print(f"[!] Warning: Expected {server_bin} was not found.")
|
||||
raise RuntimeError(f"Expected Linux server binary was not produced: {server_bin}")
|
||||
|
||||
for item in ["install_linux_client.sh", "install_linux_server.sh", "LOGAR-Linux-Client.tar.gz", "LOGAR-Linux-Server.tar.gz"]:
|
||||
item_path = os.path.join(DIST_DIR, item)
|
||||
|
||||
@@ -6,4 +6,8 @@ uvicorn>=0.28.0
|
||||
websockets>=11.0.0
|
||||
pydantic>=2.6.0
|
||||
distlib>=0.3.8
|
||||
pyinstaller>=6.0.0
|
||||
sqlalchemy>=2.0.0
|
||||
requests>=2.28.0
|
||||
httpx>=0.27.0
|
||||
pywin32>=306; sys_platform == "win32"
|
||||
|
||||
+8
-1
@@ -54,9 +54,16 @@ RUN pip3 install --no-cache-dir \
|
||||
pgpy \
|
||||
fastapi \
|
||||
uvicorn \
|
||||
websockets \
|
||||
pydantic \
|
||||
requests \
|
||||
# Code quality / linter
|
||||
httpx \
|
||||
# Database drivers & ORM for multi-engine storage
|
||||
sqlalchemy \
|
||||
psycopg2-binary \
|
||||
pymysql \
|
||||
# Code quality / linter / testing
|
||||
coverage \
|
||||
flake8
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ This directory contains the Docker configuration for the custom **LOGAR Build &
|
||||
- **Build Essentials**:
|
||||
- `gcc`, `make`, `binutils`, `build-essential`, `libffi-dev`, `libssl-dev`
|
||||
- **Pre-installed Python Packages**:
|
||||
- `pyinstaller`, `distlib`, `cryptography`, `pefile`, `pgpy`, `fastapi`, `uvicorn`, `pydantic`, `requests`
|
||||
- `pyinstaller`, `distlib`, `cryptography`, `pefile`, `pgpy`, `fastapi`, `uvicorn`, `websockets`, `pydantic`, `requests`, `sqlalchemy`, `psycopg2-binary`, `pymysql`
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
LOGAR Master License Authority Proxy
|
||||
Delegates directly to the standalone submodule: LOGAR-LICENSING/License_Authority.py
|
||||
Repo: https://gitea.eibl.tech/me0nline/LOGAR-LICENSING.git
|
||||
"""
|
||||
|
||||
import os
|
||||
import importlib.util
|
||||
|
||||
ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||
SUBMODULE_FILE = os.path.join(ROOT_DIR, "LOGAR-LICENSING", "License_Authority.py")
|
||||
|
||||
if not os.path.exists(SUBMODULE_FILE):
|
||||
raise ImportError(f"LOGAR-LICENSING submodule not found at {SUBMODULE_FILE}. Run 'git submodule update --init --recursive'.")
|
||||
|
||||
spec = importlib.util.spec_from_file_location("_submodule_authority", SUBMODULE_FILE)
|
||||
_sub = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(_sub)
|
||||
|
||||
# Export public symbols from submodule into this module's namespace
|
||||
for attr in dir(_sub):
|
||||
if not attr.startswith("__"):
|
||||
globals()[attr] = getattr(_sub, attr)
|
||||
|
||||
if __name__ == "__main__":
|
||||
_sub.main()
|
||||
+7
-132
@@ -1,139 +1,14 @@
|
||||
"""
|
||||
LOGAR Linux Edge Log Forwarder
|
||||
Collects Linux systemd journalctl / syslog entries and securely streams them
|
||||
to the central LOGAR Hub via mutual TLS (mTLS 1.3) or OpenPGP envelope.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import argparse
|
||||
import warnings
|
||||
from typing import Optional
|
||||
|
||||
# Ensure script directory and repo root are in sys.path
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
if SCRIPT_DIR not in sys.path:
|
||||
sys.path.insert(0, SCRIPT_DIR)
|
||||
REPO_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, ".."))
|
||||
if REPO_ROOT not in sys.path:
|
||||
sys.path.insert(0, REPO_ROOT)
|
||||
_CUR = os.path.dirname(__file__)
|
||||
_ROOT = os.path.abspath(os.path.join(_CUR, ".."))
|
||||
|
||||
# Suppress cryptography / pgpy deprecation notices
|
||||
warnings.filterwarnings("ignore")
|
||||
for _p in [_CUR, _ROOT]:
|
||||
if _p and _p not in sys.path:
|
||||
sys.path.insert(0, _p)
|
||||
|
||||
import pgpy
|
||||
from src.clients.Linux_Client import *
|
||||
|
||||
from log_collector import (
|
||||
get_machine_identifier,
|
||||
is_path_writable,
|
||||
get_default_data_dir,
|
||||
resolve_writable_path,
|
||||
get_state_path,
|
||||
load_state,
|
||||
save_state,
|
||||
commit_state,
|
||||
LinuxLogCollector,
|
||||
BaseLogCollector
|
||||
)
|
||||
from client_transport import (
|
||||
is_cert_expiring_soon,
|
||||
enroll_client_if_needed,
|
||||
get_tls_socket,
|
||||
load_config,
|
||||
ensure_enrolled as transport_ensure_enrolled,
|
||||
send_encrypted_logs_over_socket as transport_send_encrypted_logs
|
||||
)
|
||||
|
||||
CONFIG_FILE_NAME = "client_config.json"
|
||||
STATE_FILE_NAME = "client_state.json"
|
||||
|
||||
|
||||
def get_recent_linux_logs(hours: int = 24, state: Optional[dict] = None) -> list:
|
||||
"""
|
||||
Collects info, warnings, and errors from systemd journalctl over the lookback window.
|
||||
Edge Filtering: Retains INFO, WARNING, and ERROR. Strips DEBUG noise.
|
||||
State Tracking: Skips events older than lookback window and already sent cursors.
|
||||
"""
|
||||
collector = LinuxLogCollector(state=state)
|
||||
return collector.collect_logs(hours=hours)
|
||||
|
||||
|
||||
def send_encrypted_logs_over_socket(config: dict, logs: list):
|
||||
"""Streams logs to the LOGAR hub over mTLS or encrypted TCP envelope."""
|
||||
return transport_send_encrypted_logs(config=config, logs=logs, os_type="linux")
|
||||
|
||||
|
||||
def ensure_enrolled(config: dict, machine_id: str) -> bool:
|
||||
"""Proactively ensures client certificate enrollment with the Hub if enrollment_secret is available."""
|
||||
return transport_ensure_enrolled(config=config, machine_id=machine_id, os_type="linux")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="LOGAR Linux Edge Log Forwarder with State Tracking")
|
||||
parser.add_argument("--config", default=CONFIG_FILE_NAME, help="Path to client_config.json")
|
||||
parser.add_argument("--hours", type=int, default=24, help="Lookback window in hours for journalctl logs (default: 24)")
|
||||
parser.add_argument("--state-file", default=None, help="Path to state tracking file (default: client_state.json next to config)")
|
||||
parser.add_argument("--no-state", action="store_true", help="Disable state tracking and send all events matching lookback window")
|
||||
parser.add_argument("--interval", type=int, default=0, help="Continuous polling interval in seconds (0 = run once and exit, >0 = service loop)")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
config = load_config(args.config)
|
||||
except Exception as e:
|
||||
print(f"[!] Configuration error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
state_path = get_state_path(args.config, args.state_file)
|
||||
state = None if args.no_state else load_state(state_path)
|
||||
|
||||
machine_id = config.get("client_id") or get_machine_identifier()
|
||||
print(f"[*] Edge Forwarder Node: {machine_id}")
|
||||
if state and ("last_cursor" in state or "last_timestamp_us" in state):
|
||||
print(f"[*] State tracking active: resuming after previous cursor/timestamp (state file: {state_path})")
|
||||
elif not args.no_state:
|
||||
print(f"[*] State tracking initialized (state file: {state_path})")
|
||||
|
||||
# Proactively ensure dynamic PKI certificate enrollment with Server Hub
|
||||
ensure_enrolled(config, machine_id)
|
||||
|
||||
collector = LinuxLogCollector(state=state, machine_id=machine_id)
|
||||
|
||||
def run_forwarder_cycle():
|
||||
# Continuously verify/renew PKI enrollment if server is licensed & online
|
||||
ensure_enrolled(config, machine_id)
|
||||
|
||||
print(f"[*] Scanning Linux journalctl for unsent entries (last {args.hours} hours)...")
|
||||
candidate_logs = collector.collect_logs(hours=args.hours)
|
||||
print(f"[*] Found {len(candidate_logs)} unsent candidate entries (INFO to ERROR, entries > {args.hours}h and already-sent skipped).")
|
||||
|
||||
if not candidate_logs:
|
||||
print("[*] No new unsent events to transmit.")
|
||||
if state is not None:
|
||||
commit_state(state, state_path)
|
||||
return
|
||||
|
||||
try:
|
||||
resp = send_encrypted_logs_over_socket(config, candidate_logs)
|
||||
if state is not None and resp and resp.get("status") == "success":
|
||||
commit_state(state, state_path)
|
||||
print(f"[+] State successfully committed to {state_path}")
|
||||
except Exception as e:
|
||||
print(f"[!] Failed to stream logs to server: {e}")
|
||||
if not args.interval:
|
||||
sys.exit(1)
|
||||
|
||||
if args.interval and args.interval > 0:
|
||||
print(f"[*] Starting continuous forwarder service loop (Interval: {args.interval}s)...")
|
||||
while True:
|
||||
try:
|
||||
run_forwarder_cycle()
|
||||
except Exception as e:
|
||||
print(f"[!] Error in forwarder cycle: {e}")
|
||||
time.sleep(args.interval)
|
||||
else:
|
||||
run_forwarder_cycle()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+7
-3593
File diff suppressed because it is too large
Load Diff
+7
-137
@@ -1,144 +1,14 @@
|
||||
"""
|
||||
LOGAR Windows Edge Log Forwarder
|
||||
Collects Windows Event Logs (Application/System) and securely streams them
|
||||
to the central LOGAR Hub via mutual TLS (mTLS 1.3) or OpenPGP envelope.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import argparse
|
||||
import warnings
|
||||
from typing import Optional
|
||||
|
||||
# Ensure script directory and repo root are in sys.path
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
if SCRIPT_DIR not in sys.path:
|
||||
sys.path.insert(0, SCRIPT_DIR)
|
||||
REPO_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, ".."))
|
||||
if REPO_ROOT not in sys.path:
|
||||
sys.path.insert(0, REPO_ROOT)
|
||||
_CUR = os.path.dirname(__file__)
|
||||
_ROOT = os.path.abspath(os.path.join(_CUR, ".."))
|
||||
|
||||
# Suppress cryptography / pgpy deprecation notices
|
||||
warnings.filterwarnings("ignore")
|
||||
for _p in [_CUR, _ROOT]:
|
||||
if _p and _p not in sys.path:
|
||||
sys.path.insert(0, _p)
|
||||
|
||||
import pgpy
|
||||
from src.clients.Win_Client import *
|
||||
|
||||
try:
|
||||
import win32evtlog
|
||||
except ImportError:
|
||||
win32evtlog = None
|
||||
|
||||
from log_collector import (
|
||||
get_machine_identifier,
|
||||
is_path_writable,
|
||||
get_default_data_dir,
|
||||
resolve_writable_path,
|
||||
get_state_path,
|
||||
load_state,
|
||||
save_state,
|
||||
commit_state,
|
||||
WindowsLogCollector,
|
||||
BaseLogCollector
|
||||
)
|
||||
from client_transport import (
|
||||
is_cert_expiring_soon,
|
||||
enroll_client_if_needed,
|
||||
get_tls_socket,
|
||||
load_config,
|
||||
ensure_enrolled as transport_ensure_enrolled,
|
||||
send_encrypted_logs_over_socket as transport_send_encrypted_logs
|
||||
)
|
||||
|
||||
CONFIG_FILE_NAME = "client_config.json"
|
||||
STATE_FILE_NAME = "client_state.json"
|
||||
|
||||
|
||||
def get_recent_windows_logs(hours: int = 24, state: Optional[dict] = None) -> list:
|
||||
"""
|
||||
Scans the Windows Application Event Log backwards for events within the window.
|
||||
Edge Filtering: Retains INFO, WARNING, and ERROR. Drops Audit and Debug noise.
|
||||
State Tracking: Skips events older than lookback window and already sent records.
|
||||
"""
|
||||
collector = WindowsLogCollector(state=state, channels=["Application"])
|
||||
return collector.collect_logs(hours=hours)
|
||||
|
||||
|
||||
def send_encrypted_logs_over_socket(config: dict, logs: list):
|
||||
"""Streams logs to the LOGAR hub over mTLS or encrypted TCP envelope."""
|
||||
return transport_send_encrypted_logs(config=config, logs=logs, os_type="windows")
|
||||
|
||||
|
||||
def ensure_enrolled(config: dict, machine_id: str) -> bool:
|
||||
"""Proactively ensures client certificate enrollment with the Hub if enrollment_secret is available."""
|
||||
return transport_ensure_enrolled(config=config, machine_id=machine_id, os_type="windows")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="LOGAR Windows Edge Log Forwarder with State Tracking")
|
||||
parser.add_argument("--config", default=CONFIG_FILE_NAME, help="Path to client_config.json")
|
||||
parser.add_argument("--hours", type=int, default=24, help="Lookback window in hours for event logs (default: 24)")
|
||||
parser.add_argument("--state-file", default=None, help="Path to state tracking file (default: client_state.json next to config)")
|
||||
parser.add_argument("--no-state", action="store_true", help="Disable state tracking and send all events matching lookback window")
|
||||
parser.add_argument("--interval", type=int, default=0, help="Continuous polling interval in seconds (0 = run once and exit, >0 = service loop)")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
config = load_config(args.config)
|
||||
except Exception as e:
|
||||
print(f"[!] Configuration error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
state_path = get_state_path(args.config, args.state_file)
|
||||
state = None if args.no_state else load_state(state_path)
|
||||
|
||||
machine_id = config.get("client_id") or get_machine_identifier()
|
||||
print(f"[*] Edge Forwarder Node: {machine_id}")
|
||||
if state and "last_record_number" in state:
|
||||
print(f"[*] State tracking active: resuming from record #{state['last_record_number']} (state file: {state_path})")
|
||||
elif not args.no_state:
|
||||
print(f"[*] State tracking initialized (state file: {state_path})")
|
||||
|
||||
# Proactively ensure dynamic PKI certificate enrollment with Server Hub
|
||||
ensure_enrolled(config, machine_id)
|
||||
|
||||
collector = WindowsLogCollector(state=state, machine_id=machine_id, channels=["Application"])
|
||||
|
||||
def run_forwarder_cycle():
|
||||
# Continuously verify/renew PKI enrollment if server is licensed & online
|
||||
ensure_enrolled(config, machine_id)
|
||||
|
||||
print(f"[*] Scanning Windows Application event log for unsent entries (last {args.hours} hours)...")
|
||||
candidate_logs = collector.collect_logs(hours=args.hours)
|
||||
print(f"[*] Found {len(candidate_logs)} unsent candidate entries (INFO to ERROR, entries > {args.hours}h and already-sent skipped).")
|
||||
|
||||
if not candidate_logs:
|
||||
print("[*] No new unsent events to transmit.")
|
||||
if state is not None:
|
||||
commit_state(state, state_path)
|
||||
return
|
||||
|
||||
try:
|
||||
resp = send_encrypted_logs_over_socket(config, candidate_logs)
|
||||
if state is not None and resp and resp.get("status") == "success":
|
||||
commit_state(state, state_path)
|
||||
print(f"[+] State successfully committed to {state_path}")
|
||||
except Exception as e:
|
||||
print(f"[!] Failed to stream logs to server: {e}")
|
||||
if not args.interval:
|
||||
sys.exit(1)
|
||||
|
||||
if args.interval and args.interval > 0:
|
||||
print(f"[*] Starting continuous forwarder service loop (Interval: {args.interval}s)...")
|
||||
while True:
|
||||
try:
|
||||
run_forwarder_cycle()
|
||||
except Exception as e:
|
||||
print(f"[!] Error in forwarder cycle: {e}")
|
||||
time.sleep(args.interval)
|
||||
else:
|
||||
run_forwarder_cycle()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,20 @@
|
||||
"""
|
||||
LOGAR - Central Cloud Ingestion Hub & Fleet Log Distribution System
|
||||
Modular package namespace exposing:
|
||||
- hub: Central server, web UI, storage, licensing, and PKI modules
|
||||
- clients: Windows and Linux log collector forwarders, transport, and updater modules
|
||||
- common: Versioning and shared utilities
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||
if _ROOT not in sys.path:
|
||||
sys.path.insert(0, _ROOT)
|
||||
|
||||
from src import hub
|
||||
from src import clients
|
||||
from src import common
|
||||
|
||||
__all__ = ["hub", "clients", "common"]
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
_CUR = os.path.dirname(__file__)
|
||||
_ROOT = os.path.abspath(os.path.join(_CUR, ".."))
|
||||
|
||||
for _p in [_CUR, _ROOT]:
|
||||
if _p and _p not in sys.path:
|
||||
sys.path.insert(0, _p)
|
||||
|
||||
from src.hub.auth_manager import *
|
||||
+6
-317
@@ -1,322 +1,11 @@
|
||||
"""
|
||||
LOGAR - Unified Client Transport & PKI Enrollment Module
|
||||
Encapsulates mutual TLS (mTLS 1.3), OpenPGP fallback envelope framing,
|
||||
and dynamic certificate enrollment with the LOGAR Server Hub.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import socket
|
||||
import ssl
|
||||
import struct
|
||||
import urllib.request
|
||||
import warnings
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
# Suppress cryptography / pgpy deprecation notices
|
||||
warnings.filterwarnings("ignore")
|
||||
_CUR = os.path.dirname(__file__)
|
||||
_ROOT = os.path.abspath(os.path.join(_CUR, ".."))
|
||||
|
||||
import pgpy
|
||||
from log_collector import (
|
||||
get_machine_identifier,
|
||||
is_path_writable,
|
||||
get_default_data_dir,
|
||||
resolve_writable_path
|
||||
)
|
||||
for _p in [_CUR, _ROOT]:
|
||||
if _p and _p not in sys.path:
|
||||
sys.path.insert(0, _p)
|
||||
|
||||
CONFIG_FILE_NAME = "client_config.json"
|
||||
|
||||
|
||||
def is_cert_expiring_soon(cert_path: str, threshold_days: int = 30) -> bool:
|
||||
"""Checks if client certificate at cert_path is expiring within threshold_days."""
|
||||
if not os.path.exists(cert_path):
|
||||
return True
|
||||
try:
|
||||
from cryptography import x509
|
||||
with open(cert_path, "r", encoding="utf-8") as f:
|
||||
cert = x509.load_pem_x509_certificate(f.read().encode("utf-8"))
|
||||
expiry = getattr(cert, "not_valid_after_utc", None)
|
||||
if expiry is None:
|
||||
expiry = cert.not_valid_after.replace(tzinfo=timezone.utc)
|
||||
now = datetime.now(timezone.utc)
|
||||
return expiry <= (now + timedelta(days=threshold_days))
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
|
||||
def enroll_client_if_needed(
|
||||
hub_url: str,
|
||||
enrollment_secret: str,
|
||||
cert_dir: str,
|
||||
client_id: str,
|
||||
hostname: str,
|
||||
os_type: str = "windows",
|
||||
force_renew: bool = False,
|
||||
threshold_days: int = 30
|
||||
) -> bool:
|
||||
"""Bootstraps client enrollment if certificates are missing or expiring soon."""
|
||||
os.makedirs(cert_dir, exist_ok=True)
|
||||
ca_path = os.path.join(cert_dir, "ca.crt")
|
||||
cert_path = os.path.join(cert_dir, "client.crt")
|
||||
key_path = os.path.join(cert_dir, "client.key")
|
||||
|
||||
if not force_renew and os.path.exists(ca_path) and os.path.exists(cert_path) and os.path.exists(key_path):
|
||||
if not is_cert_expiring_soon(cert_path, threshold_days=threshold_days):
|
||||
return True
|
||||
print(f"[*] Client certificate at {cert_path} is expiring within {threshold_days} days. Auto-renewing...")
|
||||
|
||||
action_name = "re-enrolling" if os.path.exists(cert_path) else "enrolling"
|
||||
print(f"[*] Bootstrapping client {action_name} with LOGAR Hub at {hub_url}...")
|
||||
enroll_endpoint = f"{hub_url.rstrip('/')}/api/client/enroll"
|
||||
payload = {
|
||||
"client_id": client_id,
|
||||
"hostname": hostname,
|
||||
"os": os_type,
|
||||
"enrollment_secret": enrollment_secret
|
||||
}
|
||||
req = urllib.request.Request(
|
||||
enroll_endpoint,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
if resp.status != 200:
|
||||
raise RuntimeError(f"Enrollment failed with status code {resp.status}")
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
with open(ca_path, "w", encoding="utf-8") as f:
|
||||
f.write(data["ca_cert"])
|
||||
with open(cert_path, "w", encoding="utf-8") as f:
|
||||
f.write(data["client_cert"])
|
||||
with open(key_path, "w", encoding="utf-8") as f:
|
||||
f.write(data["client_key"])
|
||||
|
||||
try:
|
||||
os.chmod(key_path, 0o600)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
print(f"[+] Client certificates updated successfully in {os.path.abspath(cert_dir)}")
|
||||
return True
|
||||
|
||||
|
||||
def get_tls_socket(hub_host: str, hub_port: int, cert_dir: str):
|
||||
"""Establishes an mTLS connection with the LOGAR hub using client certificates."""
|
||||
ca_path = os.path.join(cert_dir, "ca.crt")
|
||||
cert_path = os.path.join(cert_dir, "client.crt")
|
||||
key_path = os.path.join(cert_dir, "client.key")
|
||||
|
||||
if not (os.path.exists(ca_path) and os.path.exists(cert_path) and os.path.exists(key_path)):
|
||||
raise FileNotFoundError(f"mTLS certificates not found in '{cert_dir}'. Enroll client first.")
|
||||
|
||||
ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=ca_path)
|
||||
ctx.load_cert_chain(certfile=cert_path, keyfile=key_path)
|
||||
ctx.minimum_version = ssl.TLSVersion.TLSv1_3
|
||||
ctx.check_hostname = False
|
||||
|
||||
raw_sock = socket.create_connection((hub_host, hub_port), timeout=15)
|
||||
return ctx.wrap_socket(raw_sock, server_hostname=hub_host)
|
||||
|
||||
|
||||
def load_config(config_path: str = CONFIG_FILE_NAME) -> dict:
|
||||
"""Loads and normalizes client configuration JSON."""
|
||||
actual_config_path = config_path
|
||||
if not os.path.exists(actual_config_path):
|
||||
data_dir_cfg = os.path.join(get_default_data_dir("LOGAR"), os.path.basename(config_path))
|
||||
if os.path.exists(data_dir_cfg):
|
||||
actual_config_path = data_dir_cfg
|
||||
|
||||
if not os.path.exists(actual_config_path):
|
||||
raise FileNotFoundError(
|
||||
f"Client configuration file not found at: {config_path}\n"
|
||||
f"Generate one from the server using: python Server.py --create-client-config --client-out {config_path}"
|
||||
)
|
||||
with open(actual_config_path, "r", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
|
||||
# Resolve cert_dir to a writable directory
|
||||
cfg_dir = os.path.dirname(os.path.abspath(actual_config_path))
|
||||
raw_cert_dir = config.get("cert_dir", "certs")
|
||||
if os.path.isabs(raw_cert_dir) and is_path_writable(raw_cert_dir):
|
||||
config["cert_dir"] = raw_cert_dir
|
||||
else:
|
||||
candidate = os.path.join(cfg_dir, os.path.basename(raw_cert_dir))
|
||||
if is_path_writable(candidate) or is_path_writable(cfg_dir):
|
||||
config["cert_dir"] = candidate
|
||||
else:
|
||||
config["cert_dir"] = resolve_writable_path("certs", "LOGAR", "certs")
|
||||
|
||||
# If ca_cert is present in config and ca.crt doesn't exist yet, save it
|
||||
ca_cert_pem = config.get("ca_cert")
|
||||
if ca_cert_pem and isinstance(ca_cert_pem, str):
|
||||
ca_file = os.path.join(config["cert_dir"], "ca.crt")
|
||||
if not os.path.exists(ca_file):
|
||||
try:
|
||||
os.makedirs(config["cert_dir"], exist_ok=True)
|
||||
with open(ca_file, "w", encoding="utf-8") as f:
|
||||
f.write(ca_cert_pem)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def ensure_enrolled(config: dict, machine_id: str, os_type: str = "windows") -> bool:
|
||||
"""Proactively ensures client certificate enrollment with the Hub if enrollment_secret is available."""
|
||||
cert_dir = config.get("cert_dir", "certs")
|
||||
enrollment_secret = config.get("enrollment_secret")
|
||||
if not enrollment_secret:
|
||||
return False
|
||||
|
||||
server_host = config.get("server_host", "127.0.0.1")
|
||||
hermes_host = config.get("hermes_host", server_host)
|
||||
if hermes_host in ("0.0.0.0", "::", None, ""):
|
||||
hermes_host = server_host if server_host not in ("0.0.0.0", "::", None, "") else "127.0.0.1"
|
||||
hermes_port = config.get("hermes_port", 8443)
|
||||
hub_url = f"http://{hermes_host}:{hermes_port}"
|
||||
try:
|
||||
return enroll_client_if_needed(hub_url, enrollment_secret, cert_dir, machine_id, machine_id, os_type=os_type)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Initial client enrollment bootstrap with Hub at {hub_url} failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def send_encrypted_logs_over_socket(
|
||||
config: dict,
|
||||
logs: list,
|
||||
os_type: Optional[str] = None,
|
||||
machine_id: Optional[str] = None
|
||||
) -> dict:
|
||||
"""
|
||||
Streams logs to the LOGAR hub.
|
||||
Uses mutual TLS 1.3 (mTLS) with client certificates if available,
|
||||
or falls back to OpenPGP encrypted envelope over TCP socket.
|
||||
"""
|
||||
node_os = os_type or ("windows" if sys.platform == "win32" else "linux")
|
||||
server_host = config["server_host"]
|
||||
server_port = int(config["server_port"])
|
||||
cert_dir = config.get("cert_dir", "certs")
|
||||
enrollment_secret = config.get("enrollment_secret")
|
||||
actual_machine_id = machine_id or config.get("client_id") or get_machine_identifier()
|
||||
|
||||
# Attempt automatic enrollment bootstrap if certs are missing and secret is provided
|
||||
hub_url = None
|
||||
if enrollment_secret:
|
||||
hermes_host = config.get("hermes_host", server_host)
|
||||
if hermes_host in ("0.0.0.0", "::", None):
|
||||
hermes_host = server_host if server_host not in ("0.0.0.0", "::", None) else "127.0.0.1"
|
||||
hermes_port = config.get("hermes_port", 8443)
|
||||
hub_url = f"http://{hermes_host}:{hermes_port}"
|
||||
try:
|
||||
enroll_client_if_needed(hub_url, enrollment_secret, cert_dir, actual_machine_id, actual_machine_id, os_type=node_os)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Enrollment bootstrap failed: {e}")
|
||||
|
||||
ca_path = os.path.join(cert_dir, "ca.crt")
|
||||
cert_path = os.path.join(cert_dir, "client.crt")
|
||||
key_path = os.path.join(cert_dir, "client.key")
|
||||
has_mtls_certs = os.path.exists(ca_path) and os.path.exists(cert_path) and os.path.exists(key_path)
|
||||
|
||||
if not has_mtls_certs and enrollment_secret and hub_url:
|
||||
try:
|
||||
enroll_client_if_needed(hub_url, enrollment_secret, cert_dir, actual_machine_id, actual_machine_id, os_type=node_os)
|
||||
has_mtls_certs = os.path.exists(ca_path) and os.path.exists(cert_path) and os.path.exists(key_path)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Enrollment before TLS connection failed: {e}")
|
||||
|
||||
if has_mtls_certs:
|
||||
print(f"[*] Connecting to LOGAR server at {server_host}:{server_port} over mTLS (TLS 1.3)...")
|
||||
sock = None
|
||||
try:
|
||||
sock = get_tls_socket(server_host, server_port, cert_dir)
|
||||
except (ssl.SSLError, ssl.CertificateError, ConnectionResetError) as tls_err:
|
||||
if enrollment_secret and hub_url:
|
||||
print(f"[!] TLS handshake error ({tls_err}). Re-enrolling with LOGAR Hub...")
|
||||
try:
|
||||
enroll_client_if_needed(hub_url, enrollment_secret, cert_dir, actual_machine_id, actual_machine_id, os_type=node_os, force_renew=True)
|
||||
sock = get_tls_socket(server_host, server_port, cert_dir)
|
||||
except Exception as retry_err:
|
||||
print(f"[!] Re-enrollment or reconnection retry failed: {retry_err}")
|
||||
raise
|
||||
else:
|
||||
raise
|
||||
|
||||
with sock:
|
||||
payload = {
|
||||
"server": actual_machine_id,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"logs": logs
|
||||
}
|
||||
payload_bytes = json.dumps(payload).encode("utf-8")
|
||||
frame = struct.pack(">I", len(payload_bytes)) + payload_bytes
|
||||
sock.sendall(frame)
|
||||
|
||||
resp_len_bytes = sock.recv(4)
|
||||
if not resp_len_bytes:
|
||||
raise ConnectionError("Server closed mTLS connection without response.")
|
||||
resp_len = struct.unpack(">I", resp_len_bytes)[0]
|
||||
resp_bytes = bytearray()
|
||||
while len(resp_bytes) < resp_len:
|
||||
chunk = sock.recv(min(4096, resp_len - len(resp_bytes)))
|
||||
if not chunk:
|
||||
break
|
||||
resp_bytes.extend(chunk)
|
||||
|
||||
response = json.loads(resp_bytes.decode("utf-8"))
|
||||
print(f"[+] Server response: {response}")
|
||||
return response
|
||||
|
||||
# Fallback to OpenPGP envelope over plain TCP socket
|
||||
auth_token = config.get("auth_token", "")
|
||||
pub_key_armored = config.get("server_public_key")
|
||||
if not pub_key_armored:
|
||||
raise ValueError("No server public key or mTLS certificates available for connection.")
|
||||
|
||||
expected_fp = config.get("server_fingerprint", "").replace(" ", "").upper()
|
||||
pub_key, _ = pgpy.PGPKey.from_blob(pub_key_armored)
|
||||
actual_fp = str(pub_key.fingerprint).replace(" ", "").upper()
|
||||
if expected_fp and actual_fp != expected_fp:
|
||||
raise ValueError(f"Server fingerprint mismatch! Expected {expected_fp}, but key has {actual_fp}.")
|
||||
|
||||
payload = {
|
||||
"server": actual_machine_id,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"logs": logs
|
||||
}
|
||||
payload_json = json.dumps(payload)
|
||||
pgp_msg = pgpy.PGPMessage.new(payload_json)
|
||||
encrypted_msg = pub_key.encrypt(pgp_msg)
|
||||
encrypted_armored = str(encrypted_msg)
|
||||
|
||||
envelope = {
|
||||
"auth_token": auth_token,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"encrypted_payload": encrypted_armored
|
||||
}
|
||||
envelope_bytes = json.dumps(envelope).encode("utf-8")
|
||||
|
||||
print(f"[*] Connecting to LOGAR server at {server_host}:{server_port} over secure TCP socket...")
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.settimeout(15.0)
|
||||
sock.connect((server_host, server_port))
|
||||
|
||||
frame = struct.pack(">I", len(envelope_bytes)) + envelope_bytes
|
||||
sock.sendall(frame)
|
||||
|
||||
resp_len_bytes = sock.recv(4)
|
||||
if not resp_len_bytes:
|
||||
raise ConnectionError("Server closed connection without response.")
|
||||
|
||||
resp_len = struct.unpack(">I", resp_len_bytes)[0]
|
||||
resp_bytes = bytearray()
|
||||
while len(resp_bytes) < resp_len:
|
||||
chunk = sock.recv(min(4096, resp_len - len(resp_bytes)))
|
||||
if not chunk:
|
||||
break
|
||||
resp_bytes.extend(chunk)
|
||||
|
||||
response = json.loads(resp_bytes.decode("utf-8"))
|
||||
print(f"[+] Server response: {response}")
|
||||
return response
|
||||
from src.clients.client_transport import *
|
||||
@@ -0,0 +1,11 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
_CUR = os.path.dirname(__file__)
|
||||
_ROOT = os.path.abspath(os.path.join(_CUR, ".."))
|
||||
|
||||
for _p in [_CUR, _ROOT]:
|
||||
if _p and _p not in sys.path:
|
||||
sys.path.insert(0, _p)
|
||||
|
||||
from src.clients.client_updater import *
|
||||
@@ -0,0 +1,169 @@
|
||||
"""
|
||||
LOGAR Linux Edge Log Forwarder
|
||||
Collects Linux systemd journalctl / syslog entries and securely streams them
|
||||
to the central LOGAR Hub via mutual TLS (mTLS 1.3) or OpenPGP envelope.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import argparse
|
||||
import warnings
|
||||
from typing import Optional
|
||||
|
||||
# Ensure script directory and repo root are in sys.path
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
if SCRIPT_DIR not in sys.path:
|
||||
sys.path.insert(0, SCRIPT_DIR)
|
||||
REPO_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, ".."))
|
||||
if REPO_ROOT not in sys.path:
|
||||
sys.path.insert(0, REPO_ROOT)
|
||||
|
||||
# Suppress cryptography / pgpy deprecation notices
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
import pgpy
|
||||
|
||||
try:
|
||||
from src.clients.log_collector import (
|
||||
get_machine_identifier,
|
||||
is_path_writable,
|
||||
get_default_data_dir,
|
||||
resolve_writable_path,
|
||||
get_state_path,
|
||||
load_state,
|
||||
save_state,
|
||||
commit_state,
|
||||
LinuxLogCollector,
|
||||
BaseLogCollector
|
||||
)
|
||||
from src.clients.client_transport import (
|
||||
is_cert_expiring_soon,
|
||||
enroll_client_if_needed,
|
||||
get_tls_socket,
|
||||
load_config,
|
||||
ensure_enrolled as transport_ensure_enrolled,
|
||||
send_encrypted_logs_over_socket as transport_send_encrypted_logs,
|
||||
check_and_apply_auto_update
|
||||
)
|
||||
except ImportError:
|
||||
from log_collector import (
|
||||
get_machine_identifier,
|
||||
is_path_writable,
|
||||
get_default_data_dir,
|
||||
resolve_writable_path,
|
||||
get_state_path,
|
||||
load_state,
|
||||
save_state,
|
||||
commit_state,
|
||||
LinuxLogCollector,
|
||||
BaseLogCollector
|
||||
)
|
||||
from client_transport import (
|
||||
is_cert_expiring_soon,
|
||||
enroll_client_if_needed,
|
||||
get_tls_socket,
|
||||
load_config,
|
||||
ensure_enrolled as transport_ensure_enrolled,
|
||||
send_encrypted_logs_over_socket as transport_send_encrypted_logs,
|
||||
check_and_apply_auto_update
|
||||
)
|
||||
|
||||
CONFIG_FILE_NAME = "client_config.json"
|
||||
STATE_FILE_NAME = "client_state.json"
|
||||
|
||||
|
||||
def get_recent_linux_logs(hours: int = 24, state: Optional[dict] = None) -> list:
|
||||
"""
|
||||
Collects info, warnings, and errors from systemd journalctl over the lookback window.
|
||||
Edge Filtering: Retains INFO, WARNING, and ERROR. Strips DEBUG noise.
|
||||
State Tracking: Skips events older than lookback window and already sent cursors.
|
||||
"""
|
||||
collector = LinuxLogCollector(state=state)
|
||||
return collector.collect_logs(hours=hours)
|
||||
|
||||
|
||||
def send_encrypted_logs_over_socket(config: dict, logs: list):
|
||||
"""Streams logs to the LOGAR hub over mTLS or encrypted TCP envelope."""
|
||||
return transport_send_encrypted_logs(config=config, logs=logs, os_type="linux")
|
||||
|
||||
|
||||
def ensure_enrolled(config: dict, machine_id: str) -> bool:
|
||||
"""Proactively ensures client certificate enrollment with the Hub if enrollment_secret is available."""
|
||||
return transport_ensure_enrolled(config=config, machine_id=machine_id, os_type="linux")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="LOGAR Linux Edge Log Forwarder with State Tracking")
|
||||
parser.add_argument("--config", default=CONFIG_FILE_NAME, help="Path to client_config.json")
|
||||
parser.add_argument("--hours", type=int, default=24, help="Lookback window in hours for journalctl logs (default: 24)")
|
||||
parser.add_argument("--state-file", default=None, help="Path to state tracking file (default: client_state.json next to config)")
|
||||
parser.add_argument("--no-state", action="store_true", help="Disable state tracking and send all events matching lookback window")
|
||||
parser.add_argument("--interval", type=int, default=0, help="Continuous polling interval in seconds (0 = run once and exit, >0 = service loop)")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
config = load_config(args.config)
|
||||
except Exception as e:
|
||||
print(f"[!] Configuration error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
state_path = get_state_path(args.config, args.state_file)
|
||||
state = None if args.no_state else load_state(state_path)
|
||||
|
||||
machine_id = config.get("client_id") or get_machine_identifier()
|
||||
print(f"[*] Edge Forwarder Node: {machine_id}")
|
||||
if state and ("last_cursor" in state or "last_timestamp_us" in state):
|
||||
print(f"[*] State tracking active: resuming after previous cursor/timestamp (state file: {state_path})")
|
||||
elif not args.no_state:
|
||||
print(f"[*] State tracking initialized (state file: {state_path})")
|
||||
|
||||
# Proactively ensure dynamic PKI certificate enrollment with Server Hub
|
||||
ensure_enrolled(config, machine_id)
|
||||
|
||||
collector = LinuxLogCollector(state=state, machine_id=machine_id)
|
||||
|
||||
def run_forwarder_cycle():
|
||||
# Continuously verify/renew PKI enrollment if server is licensed & online
|
||||
ensure_enrolled(config, machine_id)
|
||||
|
||||
# Periodically check server for client auto-updates
|
||||
try:
|
||||
check_and_apply_auto_update(config=config, machine_id=machine_id, os_type="linux")
|
||||
except Exception as update_err:
|
||||
pass
|
||||
|
||||
print(f"[*] Scanning Linux journalctl for unsent entries (last {args.hours} hours)...")
|
||||
candidate_logs = collector.collect_logs(hours=args.hours)
|
||||
print(f"[*] Found {len(candidate_logs)} unsent candidate entries (INFO to ERROR, entries > {args.hours}h and already-sent skipped).")
|
||||
|
||||
if not candidate_logs:
|
||||
print("[*] No new unsent events to transmit.")
|
||||
if state is not None:
|
||||
commit_state(state, state_path)
|
||||
return
|
||||
|
||||
try:
|
||||
resp = send_encrypted_logs_over_socket(config, candidate_logs)
|
||||
if state is not None and resp and resp.get("status") == "success":
|
||||
commit_state(state, state_path)
|
||||
print(f"[+] State successfully committed to {state_path}")
|
||||
except Exception as e:
|
||||
print(f"[!] Failed to stream logs to server: {e}")
|
||||
if not args.interval:
|
||||
sys.exit(1)
|
||||
|
||||
if args.interval and args.interval > 0:
|
||||
print(f"[*] Starting continuous forwarder service loop (Interval: {args.interval}s)...")
|
||||
while True:
|
||||
try:
|
||||
run_forwarder_cycle()
|
||||
except Exception as e:
|
||||
print(f"[!] Error in forwarder cycle: {e}")
|
||||
time.sleep(args.interval)
|
||||
else:
|
||||
run_forwarder_cycle()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,174 @@
|
||||
"""
|
||||
LOGAR Windows Edge Log Forwarder
|
||||
Collects Windows Event Logs (Application/System) and securely streams them
|
||||
to the central LOGAR Hub via mutual TLS (mTLS 1.3) or OpenPGP envelope.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import argparse
|
||||
import warnings
|
||||
from typing import Optional
|
||||
|
||||
# Ensure script directory and repo root are in sys.path
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
if SCRIPT_DIR not in sys.path:
|
||||
sys.path.insert(0, SCRIPT_DIR)
|
||||
REPO_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, ".."))
|
||||
if REPO_ROOT not in sys.path:
|
||||
sys.path.insert(0, REPO_ROOT)
|
||||
|
||||
# Suppress cryptography / pgpy deprecation notices
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
import pgpy
|
||||
|
||||
try:
|
||||
import win32evtlog
|
||||
except ImportError:
|
||||
win32evtlog = None
|
||||
|
||||
try:
|
||||
from src.clients.log_collector import (
|
||||
get_machine_identifier,
|
||||
is_path_writable,
|
||||
get_default_data_dir,
|
||||
resolve_writable_path,
|
||||
get_state_path,
|
||||
load_state,
|
||||
save_state,
|
||||
commit_state,
|
||||
WindowsLogCollector,
|
||||
BaseLogCollector
|
||||
)
|
||||
from src.clients.client_transport import (
|
||||
is_cert_expiring_soon,
|
||||
enroll_client_if_needed,
|
||||
get_tls_socket,
|
||||
load_config,
|
||||
ensure_enrolled as transport_ensure_enrolled,
|
||||
send_encrypted_logs_over_socket as transport_send_encrypted_logs,
|
||||
check_and_apply_auto_update
|
||||
)
|
||||
except ImportError:
|
||||
from log_collector import (
|
||||
get_machine_identifier,
|
||||
is_path_writable,
|
||||
get_default_data_dir,
|
||||
resolve_writable_path,
|
||||
get_state_path,
|
||||
load_state,
|
||||
save_state,
|
||||
commit_state,
|
||||
WindowsLogCollector,
|
||||
BaseLogCollector
|
||||
)
|
||||
from client_transport import (
|
||||
is_cert_expiring_soon,
|
||||
enroll_client_if_needed,
|
||||
get_tls_socket,
|
||||
load_config,
|
||||
ensure_enrolled as transport_ensure_enrolled,
|
||||
send_encrypted_logs_over_socket as transport_send_encrypted_logs,
|
||||
check_and_apply_auto_update
|
||||
)
|
||||
|
||||
CONFIG_FILE_NAME = "client_config.json"
|
||||
STATE_FILE_NAME = "client_state.json"
|
||||
|
||||
|
||||
def get_recent_windows_logs(hours: int = 24, state: Optional[dict] = None) -> list:
|
||||
"""
|
||||
Scans the Windows Application Event Log backwards for events within the window.
|
||||
Edge Filtering: Retains INFO, WARNING, and ERROR. Drops Audit and Debug noise.
|
||||
State Tracking: Skips events older than lookback window and already sent records.
|
||||
"""
|
||||
collector = WindowsLogCollector(state=state, channels=["Application"])
|
||||
return collector.collect_logs(hours=hours)
|
||||
|
||||
|
||||
def send_encrypted_logs_over_socket(config: dict, logs: list):
|
||||
"""Streams logs to the LOGAR hub over mTLS or encrypted TCP envelope."""
|
||||
return transport_send_encrypted_logs(config=config, logs=logs, os_type="windows")
|
||||
|
||||
|
||||
def ensure_enrolled(config: dict, machine_id: str) -> bool:
|
||||
"""Proactively ensures client certificate enrollment with the Hub if enrollment_secret is available."""
|
||||
return transport_ensure_enrolled(config=config, machine_id=machine_id, os_type="windows")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="LOGAR Windows Edge Log Forwarder with State Tracking")
|
||||
parser.add_argument("--config", default=CONFIG_FILE_NAME, help="Path to client_config.json")
|
||||
parser.add_argument("--hours", type=int, default=24, help="Lookback window in hours for event logs (default: 24)")
|
||||
parser.add_argument("--state-file", default=None, help="Path to state tracking file (default: client_state.json next to config)")
|
||||
parser.add_argument("--no-state", action="store_true", help="Disable state tracking and send all events matching lookback window")
|
||||
parser.add_argument("--interval", type=int, default=0, help="Continuous polling interval in seconds (0 = run once and exit, >0 = service loop)")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
config = load_config(args.config)
|
||||
except Exception as e:
|
||||
print(f"[!] Configuration error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
state_path = get_state_path(args.config, args.state_file)
|
||||
state = None if args.no_state else load_state(state_path)
|
||||
|
||||
machine_id = config.get("client_id") or get_machine_identifier()
|
||||
print(f"[*] Edge Forwarder Node: {machine_id}")
|
||||
if state and "last_record_number" in state:
|
||||
print(f"[*] State tracking active: resuming from record #{state['last_record_number']} (state file: {state_path})")
|
||||
elif not args.no_state:
|
||||
print(f"[*] State tracking initialized (state file: {state_path})")
|
||||
|
||||
# Proactively ensure dynamic PKI certificate enrollment with Server Hub
|
||||
ensure_enrolled(config, machine_id)
|
||||
|
||||
collector = WindowsLogCollector(state=state, machine_id=machine_id, channels=["Application"])
|
||||
|
||||
def run_forwarder_cycle():
|
||||
# Continuously verify/renew PKI enrollment if server is licensed & online
|
||||
ensure_enrolled(config, machine_id)
|
||||
|
||||
# Periodically check server for client auto-updates
|
||||
try:
|
||||
check_and_apply_auto_update(config=config, machine_id=machine_id, os_type="windows")
|
||||
except Exception as update_err:
|
||||
pass
|
||||
|
||||
print(f"[*] Scanning Windows Application event log for unsent entries (last {args.hours} hours)...")
|
||||
candidate_logs = collector.collect_logs(hours=args.hours)
|
||||
print(f"[*] Found {len(candidate_logs)} unsent candidate entries (INFO to ERROR, entries > {args.hours}h and already-sent skipped).")
|
||||
|
||||
if not candidate_logs:
|
||||
print("[*] No new unsent events to transmit.")
|
||||
if state is not None:
|
||||
commit_state(state, state_path)
|
||||
return
|
||||
|
||||
try:
|
||||
resp = send_encrypted_logs_over_socket(config, candidate_logs)
|
||||
if state is not None and resp and resp.get("status") == "success":
|
||||
commit_state(state, state_path)
|
||||
print(f"[+] State successfully committed to {state_path}")
|
||||
except Exception as e:
|
||||
print(f"[!] Failed to stream logs to server: {e}")
|
||||
if not args.interval:
|
||||
sys.exit(1)
|
||||
|
||||
if args.interval and args.interval > 0:
|
||||
print(f"[*] Starting continuous forwarder service loop (Interval: {args.interval}s)...")
|
||||
while True:
|
||||
try:
|
||||
run_forwarder_cycle()
|
||||
except Exception as e:
|
||||
print(f"[!] Error in forwarder cycle: {e}")
|
||||
time.sleep(args.interval)
|
||||
else:
|
||||
run_forwarder_cycle()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
# LOGAR Clients Package
|
||||
@@ -0,0 +1,359 @@
|
||||
"""
|
||||
LOGAR - Unified Client Transport & PKI Enrollment Module
|
||||
Encapsulates mutual TLS (mTLS 1.3), OpenPGP fallback envelope framing,
|
||||
and dynamic certificate enrollment with the LOGAR Server Hub.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import socket
|
||||
import ssl
|
||||
import struct
|
||||
import urllib.request
|
||||
import warnings
|
||||
import pgpy
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
# Suppress cryptography / pgpy deprecation notices
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
try:
|
||||
from src.clients.log_collector import (
|
||||
get_machine_identifier,
|
||||
is_path_writable,
|
||||
get_default_data_dir,
|
||||
resolve_writable_path,
|
||||
collect_system_metrics
|
||||
)
|
||||
except ImportError:
|
||||
from log_collector import (
|
||||
get_machine_identifier,
|
||||
is_path_writable,
|
||||
get_default_data_dir,
|
||||
resolve_writable_path,
|
||||
collect_system_metrics
|
||||
)
|
||||
|
||||
try:
|
||||
from src.common.version import LOGAR_VERSION
|
||||
except ImportError:
|
||||
try:
|
||||
from src.version import LOGAR_VERSION
|
||||
except ImportError:
|
||||
from version import LOGAR_VERSION
|
||||
|
||||
try:
|
||||
from src.clients.client_updater import ClientUpdater
|
||||
except ImportError:
|
||||
try:
|
||||
from src.client_updater import ClientUpdater
|
||||
except ImportError:
|
||||
from client_updater import ClientUpdater
|
||||
|
||||
CONFIG_FILE_NAME = "client_config.json"
|
||||
|
||||
|
||||
def is_cert_expiring_soon(cert_path: str, threshold_days: int = 30) -> bool:
|
||||
"""Checks if client certificate at cert_path is expiring within threshold_days."""
|
||||
if not os.path.exists(cert_path):
|
||||
return True
|
||||
try:
|
||||
from cryptography import x509
|
||||
with open(cert_path, "r", encoding="utf-8") as f:
|
||||
cert = x509.load_pem_x509_certificate(f.read().encode("utf-8"))
|
||||
expiry = getattr(cert, "not_valid_after_utc", None)
|
||||
if expiry is None:
|
||||
expiry = cert.not_valid_after.replace(tzinfo=timezone.utc)
|
||||
now = datetime.now(timezone.utc)
|
||||
return expiry <= (now + timedelta(days=threshold_days))
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
|
||||
def enroll_client_if_needed(
|
||||
hub_url: str,
|
||||
enrollment_secret: str,
|
||||
cert_dir: str,
|
||||
client_id: str,
|
||||
hostname: str,
|
||||
os_type: str = "windows",
|
||||
force_renew: bool = False,
|
||||
threshold_days: int = 30
|
||||
) -> bool:
|
||||
"""Bootstraps client enrollment if certificates are missing or expiring soon."""
|
||||
os.makedirs(cert_dir, exist_ok=True)
|
||||
ca_path = os.path.join(cert_dir, "ca.crt")
|
||||
cert_path = os.path.join(cert_dir, "client.crt")
|
||||
key_path = os.path.join(cert_dir, "client.key")
|
||||
|
||||
if not force_renew and os.path.exists(ca_path) and os.path.exists(cert_path) and os.path.exists(key_path):
|
||||
if not is_cert_expiring_soon(cert_path, threshold_days=threshold_days):
|
||||
return True
|
||||
print(f"[*] Client certificate at {cert_path} is expiring within {threshold_days} days. Auto-renewing...")
|
||||
|
||||
action_name = "re-enrolling" if os.path.exists(cert_path) else "enrolling"
|
||||
print(f"[*] Bootstrapping client {action_name} with LOGAR Hub at {hub_url}...")
|
||||
enroll_endpoint = f"{hub_url.rstrip('/')}/api/client/enroll"
|
||||
payload = {
|
||||
"client_id": client_id,
|
||||
"hostname": hostname,
|
||||
"os": os_type,
|
||||
"enrollment_secret": enrollment_secret,
|
||||
"version": LOGAR_VERSION
|
||||
}
|
||||
req = urllib.request.Request(
|
||||
enroll_endpoint,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
if resp.status != 200:
|
||||
raise RuntimeError(f"Enrollment failed with status code {resp.status}")
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
with open(ca_path, "w", encoding="utf-8") as f:
|
||||
f.write(data["ca_cert"])
|
||||
with open(cert_path, "w", encoding="utf-8") as f:
|
||||
f.write(data["client_cert"])
|
||||
with open(key_path, "w", encoding="utf-8") as f:
|
||||
f.write(data["client_key"])
|
||||
|
||||
try:
|
||||
os.chmod(key_path, 0o600)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
print(f"[+] Client certificates updated successfully in {os.path.abspath(cert_dir)}")
|
||||
return True
|
||||
|
||||
|
||||
def get_tls_socket(hub_host: str, hub_port: int, cert_dir: str):
|
||||
"""Establishes an mTLS connection with the LOGAR hub using client certificates."""
|
||||
ca_path = os.path.join(cert_dir, "ca.crt")
|
||||
cert_path = os.path.join(cert_dir, "client.crt")
|
||||
key_path = os.path.join(cert_dir, "client.key")
|
||||
|
||||
if not (os.path.exists(ca_path) and os.path.exists(cert_path) and os.path.exists(key_path)):
|
||||
raise FileNotFoundError(f"mTLS certificates not found in '{cert_dir}'. Enroll client first.")
|
||||
|
||||
ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=ca_path)
|
||||
ctx.load_cert_chain(certfile=cert_path, keyfile=key_path)
|
||||
ctx.minimum_version = ssl.TLSVersion.TLSv1_3
|
||||
ctx.check_hostname = False
|
||||
|
||||
raw_sock = socket.create_connection((hub_host, hub_port), timeout=15)
|
||||
return ctx.wrap_socket(raw_sock, server_hostname=hub_host)
|
||||
|
||||
|
||||
def load_config(config_path: str = CONFIG_FILE_NAME) -> dict:
|
||||
"""Loads and normalizes client configuration JSON."""
|
||||
actual_config_path = config_path
|
||||
if not os.path.exists(actual_config_path):
|
||||
data_dir_cfg = os.path.join(get_default_data_dir("LOGAR"), os.path.basename(config_path))
|
||||
if os.path.exists(data_dir_cfg):
|
||||
actual_config_path = data_dir_cfg
|
||||
|
||||
if not os.path.exists(actual_config_path):
|
||||
raise FileNotFoundError(
|
||||
f"Client configuration file not found at: {config_path}\n"
|
||||
f"Generate one from the server using: python Server.py --create-client-config --client-out {config_path}"
|
||||
)
|
||||
with open(actual_config_path, "r", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
|
||||
# Resolve cert_dir to a writable directory
|
||||
cfg_dir = os.path.dirname(os.path.abspath(actual_config_path))
|
||||
raw_cert_dir = config.get("cert_dir", "certs")
|
||||
if os.path.isabs(raw_cert_dir) and is_path_writable(raw_cert_dir):
|
||||
config["cert_dir"] = raw_cert_dir
|
||||
else:
|
||||
candidate = os.path.join(cfg_dir, os.path.basename(raw_cert_dir))
|
||||
if is_path_writable(candidate) or is_path_writable(cfg_dir):
|
||||
config["cert_dir"] = candidate
|
||||
else:
|
||||
config["cert_dir"] = resolve_writable_path("certs", "LOGAR", "certs")
|
||||
|
||||
# If ca_cert is present in config and ca.crt doesn't exist yet, save it
|
||||
ca_cert_pem = config.get("ca_cert")
|
||||
if ca_cert_pem and isinstance(ca_cert_pem, str):
|
||||
ca_file = os.path.join(config["cert_dir"], "ca.crt")
|
||||
if not os.path.exists(ca_file):
|
||||
try:
|
||||
os.makedirs(config["cert_dir"], exist_ok=True)
|
||||
with open(ca_file, "w", encoding="utf-8") as f:
|
||||
f.write(ca_cert_pem)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def ensure_enrolled(config: dict, machine_id: str, os_type: str = "windows") -> bool:
|
||||
"""Proactively ensures client certificate enrollment with the Hub if enrollment_secret is available."""
|
||||
cert_dir = config.get("cert_dir", "certs")
|
||||
enrollment_secret = config.get("enrollment_secret")
|
||||
if not enrollment_secret:
|
||||
return False
|
||||
|
||||
server_host = config.get("server_host", "127.0.0.1")
|
||||
hermes_host = config.get("hermes_host", server_host)
|
||||
if hermes_host in ("0.0.0.0", "::", None, ""):
|
||||
hermes_host = server_host if server_host not in ("0.0.0.0", "::", None, "") else "127.0.0.1"
|
||||
hermes_port = config.get("hermes_port", 8443)
|
||||
hub_url = f"http://{hermes_host}:{hermes_port}"
|
||||
try:
|
||||
return enroll_client_if_needed(hub_url, enrollment_secret, cert_dir, machine_id, machine_id, os_type=os_type)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Initial client enrollment bootstrap with Hub at {hub_url} failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def send_encrypted_logs_over_socket(
|
||||
config: dict,
|
||||
logs: list,
|
||||
os_type: Optional[str] = None,
|
||||
machine_id: Optional[str] = None
|
||||
) -> dict:
|
||||
"""
|
||||
Streams logs to the LOGAR hub.
|
||||
Uses mutual TLS 1.3 (mTLS) with client certificates if available,
|
||||
or falls back to OpenPGP encrypted envelope over TCP socket.
|
||||
"""
|
||||
node_os = os_type or ("windows" if sys.platform == "win32" else "linux")
|
||||
server_host = config["server_host"]
|
||||
server_port = int(config["server_port"])
|
||||
cert_dir = config.get("cert_dir", "certs")
|
||||
enrollment_secret = config.get("enrollment_secret")
|
||||
actual_machine_id = machine_id or config.get("client_id") or get_machine_identifier()
|
||||
|
||||
# Attempt automatic enrollment bootstrap if certs are missing and secret is provided
|
||||
hub_url = None
|
||||
if enrollment_secret:
|
||||
hermes_host = config.get("hermes_host", server_host)
|
||||
if hermes_host in ("0.0.0.0", "::", None):
|
||||
hermes_host = server_host if server_host not in ("0.0.0.0", "::", None) else "127.0.0.1"
|
||||
hermes_port = config.get("hermes_port", 8443)
|
||||
hub_url = f"http://{hermes_host}:{hermes_port}"
|
||||
try:
|
||||
enroll_client_if_needed(hub_url, enrollment_secret, cert_dir, actual_machine_id, actual_machine_id, os_type=node_os)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Enrollment bootstrap failed: {e}")
|
||||
|
||||
ca_path = os.path.join(cert_dir, "ca.crt")
|
||||
cert_path = os.path.join(cert_dir, "client.crt")
|
||||
key_path = os.path.join(cert_dir, "client.key")
|
||||
has_mtls_certs = os.path.exists(ca_path) and os.path.exists(cert_path) and os.path.exists(key_path)
|
||||
|
||||
if not has_mtls_certs and enrollment_secret and hub_url:
|
||||
try:
|
||||
enroll_client_if_needed(hub_url, enrollment_secret, cert_dir, actual_machine_id, actual_machine_id, os_type=node_os)
|
||||
has_mtls_certs = os.path.exists(ca_path) and os.path.exists(cert_path) and os.path.exists(key_path)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Enrollment before TLS connection failed: {e}")
|
||||
|
||||
if has_mtls_certs:
|
||||
print(f"[*] Connecting to LOGAR server at {server_host}:{server_port} over mTLS (TLS 1.3)...")
|
||||
sock = None
|
||||
try:
|
||||
sock = get_tls_socket(server_host, server_port, cert_dir)
|
||||
except (ssl.SSLError, ssl.CertificateError, ConnectionResetError) as tls_err:
|
||||
if enrollment_secret and hub_url:
|
||||
print(f"[!] TLS handshake error ({tls_err}). Re-enrolling with LOGAR Hub...")
|
||||
try:
|
||||
enroll_client_if_needed(hub_url, enrollment_secret, cert_dir, actual_machine_id, actual_machine_id, os_type=node_os, force_renew=True)
|
||||
sock = get_tls_socket(server_host, server_port, cert_dir)
|
||||
except Exception as retry_err:
|
||||
print(f"[!] Re-enrollment or reconnection retry failed: {retry_err}")
|
||||
raise
|
||||
else:
|
||||
raise
|
||||
|
||||
with sock:
|
||||
payload = {
|
||||
"server": actual_machine_id,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"logs": logs,
|
||||
"metrics": collect_system_metrics(),
|
||||
"version": LOGAR_VERSION
|
||||
}
|
||||
payload_bytes = json.dumps(payload).encode("utf-8")
|
||||
frame = struct.pack(">I", len(payload_bytes)) + payload_bytes
|
||||
sock.sendall(frame)
|
||||
|
||||
resp_len_bytes = sock.recv(4)
|
||||
if not resp_len_bytes:
|
||||
raise ConnectionError("Server closed mTLS connection without response.")
|
||||
resp_len = struct.unpack(">I", resp_len_bytes)[0]
|
||||
resp_bytes = bytearray()
|
||||
while len(resp_bytes) < resp_len:
|
||||
chunk = sock.recv(min(4096, resp_len - len(resp_bytes)))
|
||||
if not chunk:
|
||||
break
|
||||
resp_bytes.extend(chunk)
|
||||
|
||||
response = json.loads(resp_bytes.decode("utf-8"))
|
||||
print(f"[+] Server response: {response}")
|
||||
return response
|
||||
|
||||
# Fallback to OpenPGP envelope over plain TCP socket
|
||||
auth_token = config.get("auth_token", "")
|
||||
pub_key_armored = config.get("server_public_key")
|
||||
if not pub_key_armored:
|
||||
raise ValueError("No server public key or mTLS certificates available for connection.")
|
||||
|
||||
expected_fp = config.get("server_fingerprint", "").replace(" ", "").upper()
|
||||
pub_key, _ = pgpy.PGPKey.from_blob(pub_key_armored)
|
||||
actual_fp = str(pub_key.fingerprint).replace(" ", "").upper()
|
||||
if expected_fp and actual_fp != expected_fp:
|
||||
raise ValueError(f"Server fingerprint mismatch! Expected {expected_fp}, but key has {actual_fp}.")
|
||||
|
||||
payload = {
|
||||
"server": actual_machine_id,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"logs": logs,
|
||||
"metrics": collect_system_metrics(),
|
||||
"version": LOGAR_VERSION
|
||||
}
|
||||
payload_json = json.dumps(payload)
|
||||
pgp_msg = pgpy.PGPMessage.new(payload_json)
|
||||
encrypted_msg = pub_key.encrypt(pgp_msg)
|
||||
encrypted_armored = str(encrypted_msg)
|
||||
|
||||
envelope = {
|
||||
"auth_token": auth_token,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"encrypted_payload": encrypted_armored
|
||||
}
|
||||
envelope_bytes = json.dumps(envelope).encode("utf-8")
|
||||
|
||||
print(f"[*] Connecting to LOGAR server at {server_host}:{server_port} over secure TCP socket...")
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.settimeout(15.0)
|
||||
sock.connect((server_host, server_port))
|
||||
|
||||
frame = struct.pack(">I", len(envelope_bytes)) + envelope_bytes
|
||||
sock.sendall(frame)
|
||||
|
||||
resp_len_bytes = sock.recv(4)
|
||||
if not resp_len_bytes:
|
||||
raise ConnectionError("Server closed connection without response.")
|
||||
|
||||
resp_len = struct.unpack(">I", resp_len_bytes)[0]
|
||||
resp_bytes = bytearray()
|
||||
while len(resp_bytes) < resp_len:
|
||||
chunk = sock.recv(min(4096, resp_len - len(resp_bytes)))
|
||||
if not chunk:
|
||||
break
|
||||
resp_bytes.extend(chunk)
|
||||
|
||||
response = json.loads(resp_bytes.decode("utf-8"))
|
||||
print(f"[+] Server response: {response}")
|
||||
return response
|
||||
|
||||
|
||||
def check_and_apply_auto_update(config: dict, machine_id: str, os_type: Optional[str] = None, auto_apply: bool = True) -> Dict[str, Any]:
|
||||
"""Helper to check and apply auto updates for edge forwarders."""
|
||||
updater = ClientUpdater(config=config, os_type=os_type)
|
||||
return updater.check_and_apply_auto_update(client_id=machine_id, auto_apply=auto_apply)
|
||||
@@ -0,0 +1,248 @@
|
||||
"""
|
||||
LOGAR - Client Auto-Update Agent Module
|
||||
Handles checking the LOGAR Server Hub for client updates, downloading verified binaries (SHA-256),
|
||||
and applying atomic in-place updates across Windows and Linux forwarders.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import shutil
|
||||
import hashlib
|
||||
import tempfile
|
||||
import subprocess
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
from typing import Dict, Any, Optional, Tuple
|
||||
|
||||
try:
|
||||
from src.version import LOGAR_VERSION, is_newer_version
|
||||
except ImportError:
|
||||
from version import LOGAR_VERSION, is_newer_version
|
||||
|
||||
|
||||
class ClientUpdater:
|
||||
def __init__(self, config: dict, os_type: Optional[str] = None):
|
||||
self.config = config
|
||||
self.os_type = (os_type or ("windows" if sys.platform == "win32" else "linux")).lower()
|
||||
self.current_version = LOGAR_VERSION
|
||||
|
||||
@staticmethod
|
||||
def calculate_sha256(filepath: str) -> str:
|
||||
"""Computes SHA-256 digest of a local file."""
|
||||
if not os.path.exists(filepath):
|
||||
return ""
|
||||
h = hashlib.sha256()
|
||||
with open(filepath, "rb") as f:
|
||||
while chunk := f.read(65536):
|
||||
h.update(chunk)
|
||||
return h.hexdigest().lower()
|
||||
|
||||
def get_server_base_url(self) -> str:
|
||||
"""Resolves the HTTP base URL for server management endpoints."""
|
||||
server_host = self.config.get("server_host", "127.0.0.1")
|
||||
hermes_host = self.config.get("hermes_host", server_host)
|
||||
if hermes_host in ("0.0.0.0", "::", None, ""):
|
||||
hermes_host = server_host if server_host not in ("0.0.0.0", "::", None, "") else "127.0.0.1"
|
||||
hermes_port = int(self.config.get("hermes_port", 8443))
|
||||
return f"http://{hermes_host}:{hermes_port}"
|
||||
|
||||
def check_for_update(self, client_id: str, timeout: int = 6) -> Dict[str, Any]:
|
||||
"""
|
||||
Queries the server Hub to check if a newer client binary is available.
|
||||
"""
|
||||
base_url = self.get_server_base_url()
|
||||
url = (
|
||||
f"{base_url}/api/client/update/check"
|
||||
f"?client_id={urllib.parse.quote(client_id)}"
|
||||
f"&os={urllib.parse.quote(self.os_type)}"
|
||||
f"¤t_version={urllib.parse.quote(self.current_version)}"
|
||||
)
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={"User-Agent": f"LOGAR-Client/{self.current_version} ({self.os_type})"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
if resp.status == 200:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
except Exception as e:
|
||||
# Silent fallback during standard forwarder operation
|
||||
pass
|
||||
return {"update_available": False, "latest_version": self.current_version}
|
||||
|
||||
def download_package(self, download_path_rel: str, expected_sha256: str, target_temp: str, timeout: int = 30) -> bool:
|
||||
"""
|
||||
Downloads the client binary package from the server and validates its SHA-256 checksum.
|
||||
Uses local binary hash checking and ETag conditional headers to avoid redundant downloads.
|
||||
"""
|
||||
# 1. If running binary already matches target SHA-256, skip downloading
|
||||
target_exec = sys.executable
|
||||
if expected_sha256 and os.path.exists(target_exec):
|
||||
try:
|
||||
local_sha = self.calculate_sha256(target_exec)
|
||||
if local_sha and local_sha == expected_sha256.lower():
|
||||
print(f"[*] Local binary already matches target checksum ({local_sha[:12]}...). Skipping redundant download.")
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
base_url = self.get_server_base_url()
|
||||
download_url = f"{base_url.rstrip('/')}/{download_path_rel.lstrip('/')}"
|
||||
|
||||
print(f"[*] Downloading client update package from {download_url}...")
|
||||
headers = {"User-Agent": f"LOGAR-Client/{self.current_version} ({self.os_type})"}
|
||||
if expected_sha256:
|
||||
headers["If-None-Match"] = f'"{expected_sha256}"'
|
||||
|
||||
req = urllib.request.Request(download_url, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
if resp.status == 304:
|
||||
print("[+] Server returned 304 Not Modified. Local binary package is current.")
|
||||
return True
|
||||
if resp.status != 200:
|
||||
print(f"[!] Server returned HTTP {resp.status} during update download.")
|
||||
return False
|
||||
with open(target_temp, "wb") as f:
|
||||
while chunk := resp.read(65536):
|
||||
f.write(chunk)
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 304:
|
||||
print("[+] Server returned 304 Not Modified. Local binary package is current.")
|
||||
return True
|
||||
print(f"[!] HTTP error during update download: {e}")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"[!] Error downloading update package: {e}")
|
||||
return False
|
||||
|
||||
# Verify SHA-256
|
||||
actual_sha256 = self.calculate_sha256(target_temp)
|
||||
if expected_sha256 and actual_sha256 != expected_sha256.lower():
|
||||
print(f"[!] Update verification failed: Checksum mismatch! Expected: {expected_sha256}, Got: {actual_sha256}")
|
||||
if os.path.exists(target_temp):
|
||||
os.remove(target_temp)
|
||||
return False
|
||||
|
||||
print(f"[+] Download verified successfully (SHA-256: {actual_sha256[:16]}...).")
|
||||
return True
|
||||
|
||||
def apply_update_linux(self, new_binary_path: str, target_binary_path: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Applies update on Linux by setting permissions and atomically moving the binary.
|
||||
"""
|
||||
target = target_binary_path or sys.executable
|
||||
try:
|
||||
os.chmod(new_binary_path, 0o755)
|
||||
# Atomically replace target binary
|
||||
os.replace(new_binary_path, target)
|
||||
print(f"[+] Successfully replaced client binary: {target}")
|
||||
|
||||
# If running as a systemd service, restart service
|
||||
if shutil.which("systemctl"):
|
||||
try:
|
||||
subprocess.run(["systemctl", "restart", "logar-client"], check=False)
|
||||
print("[+] Signaled systemd to restart logar-client service.")
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"[!] Failed to apply Linux update: {e}")
|
||||
return False
|
||||
|
||||
def apply_update_windows(self, new_binary_path: str, target_binary_path: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Applies update on Windows. Since the currently running .exe cannot be directly overwritten,
|
||||
spawns a detached background helper script that waits for current PID to exit, swaps the binary,
|
||||
and restarts the service or executable.
|
||||
"""
|
||||
target = target_binary_path or sys.executable
|
||||
pid = os.getpid()
|
||||
target_dir = os.path.dirname(os.path.abspath(target))
|
||||
updater_bat = os.path.join(target_dir, f"_update_{pid}.bat")
|
||||
|
||||
bat_script = f"""@echo off
|
||||
setlocal
|
||||
echo [*] LOGAR Client Auto-Updater waiting for process {pid} to terminate...
|
||||
:WAIT_LOOP
|
||||
tasklist /FI "PID eq {pid}" | findstr "{pid}" >nul
|
||||
if %ERRORLEVEL% == 0 (
|
||||
timeout /t 1 /nobreak >nul
|
||||
goto WAIT_LOOP
|
||||
)
|
||||
|
||||
echo [*] Applying update to {target}...
|
||||
move /Y "{new_binary_path}" "{target}" >nul
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
copy /Y "{new_binary_path}" "{target}" >nul
|
||||
)
|
||||
|
||||
if exist "{os.path.join(target_dir, 'nssm.exe')}" (
|
||||
"{os.path.join(target_dir, 'nssm.exe')}" restart LOGAR-Client >nul 2>&1
|
||||
) else (
|
||||
start "" "{target}"
|
||||
)
|
||||
|
||||
del "{new_binary_path}" 2>nul
|
||||
(goto) 2>nul & del "%~f0"
|
||||
"""
|
||||
try:
|
||||
with open(updater_bat, "w", encoding="utf-8") as f:
|
||||
f.write(bat_script)
|
||||
|
||||
# Spawn detached batch process
|
||||
DETACHED_PROCESS = 0x00000008
|
||||
CREATE_NEW_PROCESS_GROUP = 0x00000200
|
||||
subprocess.Popen(
|
||||
["cmd.exe", "/c", updater_bat],
|
||||
creationflags=DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP,
|
||||
close_fds=True
|
||||
)
|
||||
print(f"[+] Detached updater script launched ({updater_bat}). Terminating for restart...")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"[!] Failed to launch Windows updater script: {e}")
|
||||
return False
|
||||
|
||||
def check_and_apply_auto_update(self, client_id: str, auto_apply: bool = True) -> Dict[str, Any]:
|
||||
"""
|
||||
Main entry point for client periodic auto-update check.
|
||||
Checks for update, downloads, verifies, and triggers update if available.
|
||||
"""
|
||||
update_info = self.check_for_update(client_id)
|
||||
if not update_info.get("update_available"):
|
||||
return {"status": "up_to_date", "current_version": self.current_version}
|
||||
|
||||
target_ver = update_info.get("latest_version")
|
||||
download_url = update_info.get("download_url")
|
||||
expected_sha256 = update_info.get("sha256", "")
|
||||
|
||||
print(f"[!] 🔔 New LOGAR Client update available: v{self.current_version} -> v{target_ver}")
|
||||
|
||||
if not auto_apply or not download_url:
|
||||
return {"status": "available", "latest_version": target_ver}
|
||||
|
||||
# Prepare temporary file for download
|
||||
temp_dir = os.path.dirname(os.path.abspath(sys.executable))
|
||||
if not os.access(temp_dir, os.W_OK):
|
||||
temp_dir = tempfile.gettempdir()
|
||||
|
||||
suffix = ".exe" if self.os_type == "windows" else ".bin"
|
||||
temp_download = os.path.join(temp_dir, f"_update_pkg_{int(time.time())}{suffix}")
|
||||
|
||||
ok = self.download_package(download_url, expected_sha256, temp_download)
|
||||
if not ok:
|
||||
return {"status": "download_failed"}
|
||||
|
||||
print(f"[*] Applying client update to v{target_ver}...")
|
||||
if self.os_type == "windows":
|
||||
applied = self.apply_update_windows(temp_download)
|
||||
if applied and getattr(sys, "frozen", False):
|
||||
sys.exit(0)
|
||||
else:
|
||||
applied = self.apply_update_linux(temp_download)
|
||||
|
||||
return {"status": "updated" if applied else "apply_failed", "latest_version": target_ver}
|
||||
@@ -0,0 +1,679 @@
|
||||
"""
|
||||
LOGAR - Unified Cross-Platform Log Collector & State Management
|
||||
Provides modular log collection classes and state management for Windows and Linux,
|
||||
reusable across Client Agents (Win_Client, Linux_Client) and Server Hubs (Windows & Linux).
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import socket
|
||||
import shutil
|
||||
import ctypes
|
||||
import subprocess
|
||||
import warnings
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Optional, List, Dict, Any, Set
|
||||
|
||||
# Suppress cryptography / pgpy deprecation notices
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
try:
|
||||
import win32evtlog
|
||||
except ImportError:
|
||||
win32evtlog = None
|
||||
|
||||
CONFIG_FILE_NAME = "client_config.json"
|
||||
STATE_FILE_NAME = "client_state.json"
|
||||
|
||||
|
||||
def collect_system_metrics() -> Dict[str, Any]:
|
||||
"""
|
||||
Collects cross-platform host health and resource telemetry without requiring external wheels.
|
||||
Returns:
|
||||
{
|
||||
"cpu_percent": float,
|
||||
"memory_percent": float,
|
||||
"memory_total_mb": int,
|
||||
"memory_free_mb": int,
|
||||
"disk_percent_free": float,
|
||||
"disk_total_gb": float,
|
||||
"disk_free_gb": float
|
||||
}
|
||||
"""
|
||||
metrics = {
|
||||
"cpu_percent": 0.0,
|
||||
"memory_percent": 0.0,
|
||||
"memory_total_mb": 0,
|
||||
"memory_free_mb": 0,
|
||||
"disk_percent_free": 0.0,
|
||||
"disk_total_gb": 0.0,
|
||||
"disk_free_gb": 0.0
|
||||
}
|
||||
# 1. Disk metrics
|
||||
try:
|
||||
path = "C:\\" if sys.platform == "win32" else "/"
|
||||
total, used, free = shutil.disk_usage(path)
|
||||
metrics["disk_total_gb"] = round(total / (1024 ** 3), 2)
|
||||
metrics["disk_free_gb"] = round(free / (1024 ** 3), 2)
|
||||
if total > 0:
|
||||
metrics["disk_percent_free"] = round((free / total) * 100, 1)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2. Memory metrics
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
class MEMORYSTATUSEX(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("dwLength", ctypes.c_ulong),
|
||||
("dwMemoryLoad", ctypes.c_ulong),
|
||||
("ullTotalPhys", ctypes.c_ulonglong),
|
||||
("ullAvailPhys", ctypes.c_ulonglong),
|
||||
("ullTotalPageFile", ctypes.c_ulonglong),
|
||||
("ullAvailPageFile", ctypes.c_ulonglong),
|
||||
("ullTotalVirtual", ctypes.c_ulonglong),
|
||||
("ullAvailVirtual", ctypes.c_ulonglong),
|
||||
("sullAvailExtendedVirtual", ctypes.c_ulonglong),
|
||||
]
|
||||
stat = MEMORYSTATUSEX()
|
||||
stat.dwLength = ctypes.sizeof(MEMORYSTATUSEX)
|
||||
if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)):
|
||||
metrics["memory_percent"] = float(stat.dwMemoryLoad)
|
||||
metrics["memory_total_mb"] = int(stat.ullTotalPhys / (1024 * 1024))
|
||||
metrics["memory_free_mb"] = int(stat.ullAvailPhys / (1024 * 1024))
|
||||
else:
|
||||
# Linux: read /proc/meminfo
|
||||
if os.path.exists("/proc/meminfo"):
|
||||
mem_total = 0
|
||||
mem_avail = 0
|
||||
with open("/proc/meminfo", "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
parts = line.split(":")
|
||||
if len(parts) == 2:
|
||||
k, v = parts[0].strip(), parts[1].strip().split()[0]
|
||||
if k == "MemTotal":
|
||||
mem_total = int(v)
|
||||
elif k in ("MemAvailable", "MemFree") and mem_avail == 0:
|
||||
mem_avail = int(v)
|
||||
if mem_total > 0:
|
||||
metrics["memory_total_mb"] = mem_total // 1024
|
||||
metrics["memory_free_mb"] = mem_avail // 1024
|
||||
metrics["memory_percent"] = round(((mem_total - mem_avail) / mem_total) * 100, 1)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 3. CPU metrics
|
||||
try:
|
||||
if hasattr(os, "getloadavg"):
|
||||
load1, _, _ = os.getloadavg()
|
||||
cpu_count = os.cpu_count() or 1
|
||||
metrics["cpu_percent"] = round(min(100.0, (load1 / cpu_count) * 100), 1)
|
||||
else:
|
||||
try:
|
||||
import psutil
|
||||
metrics["cpu_percent"] = float(psutil.cpu_percent(interval=None))
|
||||
except ImportError:
|
||||
metrics["cpu_percent"] = 0.0
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return metrics
|
||||
|
||||
|
||||
def is_routine_service_noise(signature: str, message: str, severity: str) -> bool:
|
||||
"""
|
||||
Identifies high-frequency benign background service start/stop cycles
|
||||
(e.g., sppsvc, edgeupdate, trustedinstaller, wuauserv, backgroundTaskHost)
|
||||
to prevent false-positive anomaly fatigue and consecutive_run metric bloat.
|
||||
"""
|
||||
if str(severity).upper() != "INFO":
|
||||
return False
|
||||
sig_low = (signature or "").lower()
|
||||
msg_low = (message or "").lower()
|
||||
routine_services = [
|
||||
"sppsvc",
|
||||
"edgeupdate",
|
||||
"trustedinstaller",
|
||||
"wuauserv",
|
||||
"backgroundtaskhost",
|
||||
"service control manager",
|
||||
"systemd",
|
||||
"cron"
|
||||
]
|
||||
is_routine_svc = any(s in sig_low or s in msg_low for s in routine_services)
|
||||
if is_routine_svc:
|
||||
lifecycle_phrases = [
|
||||
"entered the running state",
|
||||
"entered the stopped state",
|
||||
"entered the start pending state",
|
||||
"entered the stop pending state",
|
||||
"started successfully",
|
||||
"stopped successfully",
|
||||
"service stopped",
|
||||
"service started",
|
||||
"session opened",
|
||||
"session closed",
|
||||
"clean",
|
||||
"reloaded"
|
||||
]
|
||||
if any(p in msg_low for p in lifecycle_phrases):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_machine_identifier() -> str:
|
||||
"""
|
||||
Returns the fully qualified hostname of the machine sending/collecting logs.
|
||||
Cross-platform resolution:
|
||||
1. FQDN (if non-local)
|
||||
2. Windows USERDNSDOMAIN (on Windows)
|
||||
3. Linux /etc/resolv.conf domain/search directive (on Linux)
|
||||
4. Reverse DNS lookup
|
||||
5. socket.gethostname() fallback
|
||||
"""
|
||||
fqdn = socket.getfqdn()
|
||||
if fqdn and "." in fqdn and not fqdn.startswith("localhost"):
|
||||
return fqdn
|
||||
|
||||
hostname = socket.gethostname()
|
||||
|
||||
# Windows DNS domain
|
||||
if sys.platform == "win32":
|
||||
user_dns_domain = os.environ.get("USERDNSDOMAIN")
|
||||
if user_dns_domain and user_dns_domain.lower() != hostname.lower():
|
||||
return f"{hostname}.{user_dns_domain.lower()}"
|
||||
|
||||
# Linux resolv.conf search / domain
|
||||
if sys.platform.startswith("linux") or os.path.exists("/etc/resolv.conf"):
|
||||
try:
|
||||
with open("/etc/resolv.conf", "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
parts = line.strip().split()
|
||||
if parts and parts[0] in ["domain", "search"] and len(parts) > 1:
|
||||
domain = parts[1]
|
||||
if domain and not domain.startswith("."):
|
||||
return f"{hostname}.{domain}"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Reverse DNS lookup
|
||||
try:
|
||||
host_ip = socket.gethostbyname(hostname)
|
||||
canonical_name = socket.gethostbyaddr(host_ip)[0]
|
||||
if canonical_name and "." in canonical_name and not canonical_name.startswith("localhost"):
|
||||
return canonical_name
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return hostname
|
||||
|
||||
|
||||
def is_path_writable(filepath: str) -> bool:
|
||||
"""Checks if a file or directory path is actually writable by the current process."""
|
||||
try:
|
||||
if os.path.exists(filepath):
|
||||
if os.path.isdir(filepath):
|
||||
test_f = os.path.join(filepath, f".perm_{os.getpid()}")
|
||||
with open(test_f, "w") as f:
|
||||
f.write("1")
|
||||
os.remove(test_f)
|
||||
return True
|
||||
else:
|
||||
with open(filepath, "a") as f:
|
||||
pass
|
||||
return True
|
||||
else:
|
||||
parent = os.path.dirname(os.path.abspath(filepath)) or "."
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
test_f = os.path.join(parent, f".perm_{os.getpid()}")
|
||||
with open(test_f, "w") as f:
|
||||
f.write("1")
|
||||
os.remove(test_f)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def get_default_data_dir(app_name: str = "LOGAR") -> str:
|
||||
"""Returns a suitable writable application data directory."""
|
||||
if sys.platform == "win32":
|
||||
# 1. ProgramData (C:\ProgramData\LOGAR)
|
||||
prog_data = os.environ.get("ProgramData", r"C:\ProgramData")
|
||||
target_prog = os.path.join(prog_data, app_name)
|
||||
if is_path_writable(target_prog):
|
||||
return target_prog
|
||||
|
||||
# 2. LocalAppData (C:\Users\<user>\AppData\Local\LOGAR)
|
||||
local_app = os.environ.get("LOCALAPPDATA") or os.path.expanduser(r"~\AppData\Local")
|
||||
target_local = os.path.join(local_app, app_name)
|
||||
if is_path_writable(target_local):
|
||||
return target_local
|
||||
|
||||
# 3. Temp directory (%TEMP%\LOGAR)
|
||||
temp_dir = os.path.join(os.environ.get("TEMP", r"C:\Windows\Temp"), app_name)
|
||||
if is_path_writable(temp_dir):
|
||||
return temp_dir
|
||||
else:
|
||||
# Linux / Unix candidates: /var/lib/<app_name>, /tmp/<app_name>, ~/.local/share/<app_name>
|
||||
var_lib = f"/var/lib/{app_name.lower()}"
|
||||
if is_path_writable(var_lib):
|
||||
return var_lib
|
||||
user_share = os.path.expanduser(f"~/.local/share/{app_name.lower()}")
|
||||
if is_path_writable(user_share):
|
||||
return user_share
|
||||
|
||||
cwd = os.getcwd()
|
||||
if is_path_writable(cwd):
|
||||
return cwd
|
||||
return os.path.expanduser("~")
|
||||
|
||||
|
||||
def resolve_writable_path(path: str, app_name: str = "LOGAR", default_filename: str = "client_state.json") -> str:
|
||||
"""Resolves a file or directory path, ensuring it is writable or falling back to a writable data directory."""
|
||||
if not path:
|
||||
path = default_filename
|
||||
|
||||
if os.path.isabs(path) and is_path_writable(path):
|
||||
return path
|
||||
|
||||
if not os.path.isabs(path) and "Program Files" not in os.getcwd() and is_path_writable(os.getcwd()):
|
||||
return os.path.abspath(path)
|
||||
|
||||
data_dir = get_default_data_dir(app_name)
|
||||
fname = os.path.basename(path)
|
||||
return os.path.join(data_dir, fname)
|
||||
|
||||
|
||||
def get_state_path(config_path: str, custom_state_path: Optional[str] = None) -> str:
|
||||
if custom_state_path:
|
||||
return custom_state_path
|
||||
config_dir = os.path.dirname(os.path.abspath(config_path))
|
||||
if sys.platform == "win32" and ("Program Files" in config_dir or not is_path_writable(config_dir)):
|
||||
return os.path.join(get_default_data_dir("LOGAR"), STATE_FILE_NAME)
|
||||
return os.path.join(config_dir, STATE_FILE_NAME)
|
||||
|
||||
|
||||
def load_state(state_path: str) -> dict:
|
||||
"""Loads state tracking dictionary from disk."""
|
||||
if os.path.exists(state_path):
|
||||
try:
|
||||
with open(state_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Failed to read state file '{state_path}': {e}")
|
||||
return {}
|
||||
return {}
|
||||
|
||||
|
||||
def save_state(state_path: str, state: dict):
|
||||
"""Atomically saves state dictionary to disk."""
|
||||
try:
|
||||
os.makedirs(os.path.dirname(os.path.abspath(state_path)), exist_ok=True)
|
||||
temp_path = f"{state_path}.tmp"
|
||||
with open(temp_path, "w", encoding="utf-8") as f:
|
||||
json.dump(state, f, indent=2)
|
||||
os.replace(temp_path, state_path)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Could not save client state to '{state_path}': {e}")
|
||||
|
||||
|
||||
def commit_state(state: dict, state_path: str):
|
||||
"""
|
||||
Commits staged cursor/record state keys to persistent fields and writes to state_path.
|
||||
Supports both Windows record numbers and Linux cursors/timestamps.
|
||||
"""
|
||||
if "new_last_record_number" in state:
|
||||
val = state.pop("new_last_record_number")
|
||||
if val:
|
||||
state["last_record_number"] = val
|
||||
if "new_sent_record_ids" in state:
|
||||
state["sent_record_ids"] = state.pop("new_sent_record_ids")
|
||||
if "new_last_cursor" in state:
|
||||
val = state.pop("new_last_cursor")
|
||||
if val:
|
||||
state["last_cursor"] = val
|
||||
if "new_last_timestamp_us" in state:
|
||||
val = state.pop("new_last_timestamp_us")
|
||||
if val:
|
||||
state["last_timestamp_us"] = val
|
||||
if "new_sent_cursors" in state:
|
||||
state["sent_cursors"] = state.pop("new_sent_cursors")
|
||||
save_state(state_path, state)
|
||||
|
||||
|
||||
class BaseLogCollector(ABC):
|
||||
"""
|
||||
Abstract base class for platform-specific edge and hub log collectors.
|
||||
Manages state persistence, machine identifiers, lookback windows, and normalization.
|
||||
"""
|
||||
|
||||
def __init__(self, state: Optional[dict] = None, machine_id: Optional[str] = None):
|
||||
self.state = state if state is not None else {}
|
||||
self.machine_id = machine_id or get_machine_identifier()
|
||||
|
||||
@abstractmethod
|
||||
def collect_logs(self, hours: int = 24) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Collects normalized logs within the given lookback window.
|
||||
Returns a list of dicts:
|
||||
[
|
||||
{
|
||||
"server": str,
|
||||
"os_type": "windows"|"linux",
|
||||
"signature": str,
|
||||
"severity": "ERROR"|"WARNING"|"INFO",
|
||||
"message": str
|
||||
},
|
||||
...
|
||||
]
|
||||
"""
|
||||
pass
|
||||
|
||||
def commit(self, state_path: Optional[str] = None):
|
||||
"""Commits staged state keys and optionally writes to disk."""
|
||||
if state_path:
|
||||
commit_state(self.state, state_path)
|
||||
else:
|
||||
if "new_last_record_number" in self.state:
|
||||
val = self.state.pop("new_last_record_number")
|
||||
if val:
|
||||
self.state["last_record_number"] = val
|
||||
if "new_sent_record_ids" in self.state:
|
||||
self.state["sent_record_ids"] = self.state.pop("new_sent_record_ids")
|
||||
if "new_last_cursor" in self.state:
|
||||
val = self.state.pop("new_last_cursor")
|
||||
if val:
|
||||
self.state["last_cursor"] = val
|
||||
if "new_last_timestamp_us" in self.state:
|
||||
val = self.state.pop("new_last_timestamp_us")
|
||||
if val:
|
||||
self.state["last_timestamp_us"] = val
|
||||
if "new_sent_cursors" in self.state:
|
||||
self.state["sent_cursors"] = self.state.pop("new_sent_cursors")
|
||||
|
||||
|
||||
class WindowsLogCollector(BaseLogCollector):
|
||||
"""
|
||||
Windows Event Log Collector.
|
||||
Reads Application and System (or custom channels) backwards.
|
||||
Edge Filtering: Retains INFO, WARNING, and ERROR. Drops Audit and Debug noise.
|
||||
State Tracking: Skips events older than lookback window and already sent records.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
state: Optional[dict] = None,
|
||||
machine_id: Optional[str] = None,
|
||||
channels: Optional[List[str]] = None
|
||||
):
|
||||
super().__init__(state=state, machine_id=machine_id)
|
||||
self.channels = channels or ["Application"]
|
||||
|
||||
def collect_logs(self, hours: int = 24) -> List[Dict[str, Any]]:
|
||||
if win32evtlog is None:
|
||||
return []
|
||||
|
||||
logs = []
|
||||
cutoff_time = datetime.now() - timedelta(hours=hours)
|
||||
|
||||
last_record_number = 0
|
||||
sent_record_ids: Set[str] = set()
|
||||
if self.state:
|
||||
last_record_number = int(self.state.get("last_record_number", 0))
|
||||
sent_record_ids = set(self.state.get("sent_record_ids", []))
|
||||
|
||||
# Windows Event Log EventTypes:
|
||||
# 1: EVENTLOG_ERROR_TYPE -> ERROR
|
||||
# 2: EVENTLOG_WARNING_TYPE -> WARNING
|
||||
# 4: EVENTLOG_INFORMATION_TYPE -> INFO
|
||||
sev_map = {
|
||||
1: "ERROR",
|
||||
2: "WARNING",
|
||||
4: "INFO"
|
||||
}
|
||||
|
||||
newest_record_number = 0
|
||||
collected_record_ids = []
|
||||
|
||||
flags = win32evtlog.EVENTLOG_BACKWARDS_READ | win32evtlog.EVENTLOG_SEQUENTIAL_READ
|
||||
|
||||
for log_type in self.channels:
|
||||
try:
|
||||
hand = win32evtlog.OpenEventLog("localhost", log_type)
|
||||
except Exception as e:
|
||||
continue
|
||||
|
||||
while True:
|
||||
try:
|
||||
events = win32evtlog.ReadEventLog(hand, flags, 0)
|
||||
except Exception:
|
||||
break
|
||||
if not events:
|
||||
break
|
||||
|
||||
for event in events:
|
||||
rec_num = int(event.RecordNumber)
|
||||
if newest_record_number == 0:
|
||||
newest_record_number = rec_num
|
||||
|
||||
# 1. Skip entries older than lookback window
|
||||
if event.TimeGenerated < cutoff_time:
|
||||
break
|
||||
|
||||
# 2. Skip already sent events
|
||||
if last_record_number > 0 and newest_record_number >= last_record_number:
|
||||
if rec_num <= last_record_number:
|
||||
break
|
||||
|
||||
rec_id = f"{log_type}:{rec_num}:{event.TimeGenerated.isoformat()}" if len(self.channels) > 1 else f"{rec_num}:{event.TimeGenerated.isoformat()}"
|
||||
if rec_id in sent_record_ids:
|
||||
continue
|
||||
|
||||
# Filter: upload everything from INFO to ERROR
|
||||
if event.EventType in sev_map:
|
||||
msg = " ".join(event.StringInserts) if event.StringInserts else "Event Log Entry"
|
||||
sig = f"{log_type}/{event.SourceName or 'Windows-Event'}" if len(self.channels) > 1 else (event.SourceName or "Windows-Event")
|
||||
logs.append({
|
||||
"server": self.machine_id,
|
||||
"os_type": "windows",
|
||||
"signature": sig,
|
||||
"severity": sev_map[event.EventType],
|
||||
"message": msg[:2048]
|
||||
})
|
||||
collected_record_ids.append(rec_id)
|
||||
|
||||
if events[-1].TimeGenerated < cutoff_time:
|
||||
break
|
||||
if last_record_number > 0 and newest_record_number >= last_record_number and events[-1].RecordNumber <= last_record_number:
|
||||
break
|
||||
|
||||
try:
|
||||
win32evtlog.CloseEventLog(hand)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if self.state is not None:
|
||||
target_rec = max(newest_record_number, last_record_number)
|
||||
self.state["new_last_record_number"] = target_rec
|
||||
self.state["new_sent_record_ids"] = (list(sent_record_ids) + collected_record_ids)[-1000:]
|
||||
# Also update direct keys for immediate server-side ingestion
|
||||
self.state["last_record_number"] = target_rec
|
||||
self.state["sent_record_ids"] = self.state["new_sent_record_ids"]
|
||||
self.state["last_run_timestamp"] = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
return logs
|
||||
|
||||
|
||||
class LinuxLogCollector(BaseLogCollector):
|
||||
"""
|
||||
Linux Systemd Journal / Syslog Log Collector.
|
||||
Collects info, warnings, and errors from journalctl (or syslog fallback).
|
||||
Edge Filtering: Retains INFO, WARNING, and ERROR. Drops DEBUG noise.
|
||||
State Tracking: Skips events older than lookback window and already sent cursors.
|
||||
"""
|
||||
|
||||
def __init__(self, state: Optional[dict] = None, machine_id: Optional[str] = None):
|
||||
super().__init__(state=state, machine_id=machine_id)
|
||||
|
||||
def collect_logs(self, hours: int = 24) -> List[Dict[str, Any]]:
|
||||
last_cursor = None
|
||||
last_timestamp_us = 0.0
|
||||
sent_cursors: Set[str] = set()
|
||||
if self.state:
|
||||
last_cursor = self.state.get("last_cursor")
|
||||
try:
|
||||
last_timestamp_us = float(self.state.get("last_timestamp_us", 0))
|
||||
except (ValueError, TypeError):
|
||||
last_timestamp_us = 0.0
|
||||
sent_cursors = set(self.state.get("sent_cursors", []))
|
||||
|
||||
cmd = ["journalctl", "--since", f"{hours} hours ago", "-p", "info", "--output=json"]
|
||||
result = None
|
||||
|
||||
if last_cursor:
|
||||
cmd_with_cursor = ["journalctl", "--since", f"{hours} hours ago", "--after-cursor", str(last_cursor), "-p", "info", "--output=json"]
|
||||
try:
|
||||
res = subprocess.run(cmd_with_cursor, capture_output=True, text=True, check=False)
|
||||
if res.returncode == 0:
|
||||
result = res
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if result is None:
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||
except FileNotFoundError:
|
||||
# journalctl not present, attempt fallback to /var/log/syslog or messages
|
||||
return self._collect_syslog_fallback(hours=hours)
|
||||
except Exception as e:
|
||||
print(f"[!] Error running journalctl: {e}")
|
||||
return []
|
||||
|
||||
if not result or result.returncode != 0:
|
||||
return []
|
||||
|
||||
logs = []
|
||||
cutoff_epoch_us = (datetime.now(timezone.utc) - timedelta(hours=hours)).timestamp() * 1_000_000
|
||||
|
||||
newest_cursor = None
|
||||
newest_timestamp_us = last_timestamp_us
|
||||
collected_cursors = []
|
||||
|
||||
for line in result.stdout.splitlines():
|
||||
line_str = line.strip()
|
||||
if not line_str:
|
||||
continue
|
||||
try:
|
||||
entry = json.loads(line_str)
|
||||
entry_cursor = entry.get("__CURSOR")
|
||||
entry_ts_us_raw = entry.get("__REALTIME_TIMESTAMP")
|
||||
|
||||
entry_ts_us = 0.0
|
||||
if entry_ts_us_raw:
|
||||
try:
|
||||
entry_ts_us = float(entry_ts_us_raw)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# 1. Skip entries older than lookback window
|
||||
if entry_ts_us and entry_ts_us < cutoff_epoch_us:
|
||||
continue
|
||||
|
||||
# 2. Skip already sent events
|
||||
if entry_cursor and (entry_cursor in sent_cursors or entry_cursor == last_cursor):
|
||||
continue
|
||||
if last_timestamp_us > 0 and entry_ts_us > 0 and entry_ts_us < last_timestamp_us:
|
||||
continue
|
||||
|
||||
# Advance tracking
|
||||
if entry_cursor:
|
||||
newest_cursor = entry_cursor
|
||||
collected_cursors.append(entry_cursor)
|
||||
if entry_ts_us > newest_timestamp_us:
|
||||
newest_timestamp_us = entry_ts_us
|
||||
|
||||
priority = int(entry.get("PRIORITY", "6"))
|
||||
if priority > 6:
|
||||
continue
|
||||
|
||||
if priority <= 3:
|
||||
sev = "ERROR"
|
||||
elif priority in (4, 5):
|
||||
sev = "WARNING"
|
||||
else:
|
||||
sev = "INFO"
|
||||
|
||||
logs.append({
|
||||
"server": self.machine_id,
|
||||
"os_type": "linux",
|
||||
"signature": entry.get("SYSLOG_IDENTIFIER", "unknown"),
|
||||
"severity": sev,
|
||||
"message": entry.get("MESSAGE", "")[:2048]
|
||||
})
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
|
||||
if self.state is not None:
|
||||
self.state["new_last_cursor"] = newest_cursor or last_cursor
|
||||
self.state["new_last_timestamp_us"] = max(newest_timestamp_us, last_timestamp_us)
|
||||
self.state["new_sent_cursors"] = (list(sent_cursors) + collected_cursors)[-1000:]
|
||||
# Also update direct keys for immediate server-side ingestion
|
||||
self.state["last_cursor"] = self.state["new_last_cursor"]
|
||||
self.state["last_timestamp_us"] = self.state["new_last_timestamp_us"]
|
||||
self.state["sent_cursors"] = self.state["new_sent_cursors"]
|
||||
self.state["last_run_timestamp"] = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
return logs
|
||||
|
||||
def _collect_syslog_fallback(self, hours: int = 24) -> List[Dict[str, Any]]:
|
||||
"""Fallback log reader for non-systemd Linux environments."""
|
||||
candidates = ["/var/log/syslog", "/var/log/messages"]
|
||||
target = next((f for f in candidates if os.path.exists(f) and os.access(f, os.R_OK)), None)
|
||||
if not target:
|
||||
return []
|
||||
|
||||
logs = []
|
||||
try:
|
||||
with open(target, "r", encoding="utf-8", errors="ignore") as f:
|
||||
lines = f.readlines()[-500:] # Read recent 500 lines
|
||||
for line in lines:
|
||||
line_str = line.strip()
|
||||
if not line_str:
|
||||
continue
|
||||
sev = "INFO"
|
||||
lower_l = line_str.lower()
|
||||
if any(w in lower_l for w in ["err", "crit", "alert", "emerg", "fail"]):
|
||||
sev = "ERROR"
|
||||
elif any(w in lower_l for w in ["warn", "notice"]):
|
||||
sev = "WARNING"
|
||||
|
||||
logs.append({
|
||||
"server": self.machine_id,
|
||||
"os_type": "linux",
|
||||
"signature": "syslog",
|
||||
"severity": sev,
|
||||
"message": line_str[:2048]
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
return logs
|
||||
|
||||
|
||||
def get_log_collector(
|
||||
os_type: Optional[str] = None,
|
||||
state: Optional[dict] = None,
|
||||
machine_id: Optional[str] = None,
|
||||
channels: Optional[List[str]] = None
|
||||
) -> BaseLogCollector:
|
||||
"""
|
||||
Factory helper that returns the appropriate BaseLogCollector instance.
|
||||
If os_type is omitted, auto-detects based on sys.platform.
|
||||
"""
|
||||
target_os = os_type.lower() if os_type else ("windows" if sys.platform == "win32" else "linux")
|
||||
if target_os == "windows":
|
||||
chans = channels or (["Application", "System"] if channels is None else ["Application"])
|
||||
return WindowsLogCollector(state=state, machine_id=machine_id, channels=chans)
|
||||
else:
|
||||
return LinuxLogCollector(state=state, machine_id=machine_id)
|
||||
@@ -0,0 +1 @@
|
||||
# LOGAR Common Package
|
||||
@@ -0,0 +1,67 @@
|
||||
"""
|
||||
LOGAR - Unified Versioning and Semantic Comparison Module
|
||||
Provides canonical application version constants and semantic version comparison utilities.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from typing import Tuple, Optional
|
||||
|
||||
|
||||
LOGAR_VERSION = "0.3.5.2"
|
||||
__version__ = LOGAR_VERSION
|
||||
|
||||
|
||||
def parse_version(version_str: Optional[str]) -> Tuple[int, ...]:
|
||||
"""
|
||||
Parses a version string into a comparable tuple of integers.
|
||||
Strips leading 'v' or 'V' and non-numeric suffixes.
|
||||
e.g. 'v0.3.5.2' -> (0, 3, 5, 2), '0.3.5-beta' -> (0, 3, 5, 0)
|
||||
"""
|
||||
if not version_str:
|
||||
return (0, 0, 0, 0)
|
||||
|
||||
clean = str(version_str).strip()
|
||||
if clean.lower().startswith("v"):
|
||||
clean = clean[1:].strip()
|
||||
|
||||
# Extract digit sequences separated by dots
|
||||
match = re.match(r"^(\d+(?:\.\d+)*)", clean)
|
||||
if not match:
|
||||
return (0, 0, 0, 0)
|
||||
|
||||
parts = [int(p) for p in match.group(1).split(".")]
|
||||
# Pad to at least 4 components for consistent 4-part semantic version comparison
|
||||
while len(parts) < 4:
|
||||
parts.append(0)
|
||||
return tuple(parts)
|
||||
|
||||
|
||||
def compare_versions(v1: Optional[str], v2: Optional[str]) -> int:
|
||||
"""
|
||||
Compares two version strings.
|
||||
Returns:
|
||||
1 if v1 > v2 (v1 is newer than v2)
|
||||
0 if v1 == v2
|
||||
-1 if v1 < v2 (v1 is older than v2)
|
||||
"""
|
||||
t1 = parse_version(v1)
|
||||
t2 = parse_version(v2)
|
||||
if t1 > t2:
|
||||
return 1
|
||||
elif t1 < t2:
|
||||
return -1
|
||||
return 0
|
||||
|
||||
|
||||
def is_newer_version(candidate_version: Optional[str], current_version: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Returns True if candidate_version is strictly newer than current_version (or LOGAR_VERSION).
|
||||
"""
|
||||
base = current_version or LOGAR_VERSION
|
||||
return compare_versions(candidate_version, base) > 0
|
||||
|
||||
|
||||
def get_current_version() -> str:
|
||||
"""Returns the canonical current version string."""
|
||||
return LOGAR_VERSION
|
||||
+2536
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
# LOGAR Hub Package
|
||||
@@ -0,0 +1,398 @@
|
||||
"""
|
||||
LOGAR - Server Authentication & Access Control Managers
|
||||
Provides modular classes for Session Management, Rate Limiting, Password Hashing,
|
||||
Two-Factor Authentication (TOTP / RFC 6238), and Hermes API Tokens.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import base64
|
||||
import struct
|
||||
import secrets
|
||||
import hashlib
|
||||
import hmac
|
||||
import sqlite3
|
||||
import urllib.parse
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, Dict, Any, List, Tuple
|
||||
from fastapi import Request, HTTPException
|
||||
|
||||
|
||||
class SessionManager:
|
||||
"""Manages cryptographic session secrets and HMAC-signed session tokens."""
|
||||
|
||||
SESSION_COOKIE_NAME = "logar_server_session"
|
||||
_CACHED_SESSION_SECRETS: Dict[str, str] = {}
|
||||
|
||||
@classmethod
|
||||
def get_session_secret(cls, db_path: Optional[str] = None, fallback_db: str = "logar_state.db") -> str:
|
||||
"""Retrieves or persists a cryptographically secure session secret in the database."""
|
||||
env_sec = os.environ.get("LOGAR_SESSION_SECRET")
|
||||
if env_sec:
|
||||
return env_sec
|
||||
target_db = db_path or fallback_db
|
||||
abs_db = os.path.abspath(target_db)
|
||||
if abs_db in cls._CACHED_SESSION_SECRETS:
|
||||
return cls._CACHED_SESSION_SECRETS[abs_db]
|
||||
try:
|
||||
conn = sqlite3.connect(target_db, timeout=5.0)
|
||||
c = conn.cursor()
|
||||
c.execute("CREATE TABLE IF NOT EXISTS server_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
||||
c.execute("SELECT value FROM server_meta WHERE key = 'session_secret'")
|
||||
row = c.fetchone()
|
||||
if row and row[0]:
|
||||
cls._CACHED_SESSION_SECRETS[abs_db] = row[0]
|
||||
conn.close()
|
||||
return row[0]
|
||||
new_secret = secrets.token_hex(32)
|
||||
c.execute("INSERT OR REPLACE INTO server_meta (key, value) VALUES ('session_secret', ?)", (new_secret,))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
cls._CACHED_SESSION_SECRETS[abs_db] = new_secret
|
||||
return new_secret
|
||||
except Exception:
|
||||
fallback = secrets.token_hex(32)
|
||||
cls._CACHED_SESSION_SECRETS[abs_db] = fallback
|
||||
return fallback
|
||||
|
||||
@classmethod
|
||||
def generate_session_token(cls, username: str, db_path: Optional[str] = None, fallback_db: str = "logar_state.db") -> str:
|
||||
"""Generates an HMAC-signed session cookie string."""
|
||||
ts = int(datetime.now(timezone.utc).timestamp())
|
||||
nonce = secrets.token_hex(8)
|
||||
payload = f"{username}:{ts}:{nonce}"
|
||||
secret = cls.get_session_secret(db_path=db_path, fallback_db=fallback_db)
|
||||
sig = hmac.new(secret.encode("utf-8"), payload.encode("utf-8"), hashlib.sha256).hexdigest()
|
||||
return f"{payload}:{sig}"
|
||||
|
||||
@classmethod
|
||||
def verify_session_token(
|
||||
cls,
|
||||
token: Optional[str],
|
||||
max_age_hours: int = 24,
|
||||
db_path: Optional[str] = None,
|
||||
fallback_db: str = "logar_state.db"
|
||||
) -> Optional[str]:
|
||||
"""Verifies session token authenticity and age."""
|
||||
if not token or ":" not in token:
|
||||
return None
|
||||
parts = token.split(":")
|
||||
if len(parts) != 4:
|
||||
return None
|
||||
username, ts_str, nonce, sig = parts
|
||||
payload = f"{username}:{ts_str}:{nonce}"
|
||||
secret = cls.get_session_secret(db_path=db_path, fallback_db=fallback_db)
|
||||
expected_sig = hmac.new(secret.encode("utf-8"), payload.encode("utf-8"), hashlib.sha256).hexdigest()
|
||||
if not hmac.compare_digest(sig, expected_sig):
|
||||
return None
|
||||
try:
|
||||
ts = int(ts_str)
|
||||
if datetime.now(timezone.utc).timestamp() - ts > max_age_hours * 3600:
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
return username
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
"""Sliding-window IP rate limiter with persistent SQLite tracking and memory fallback."""
|
||||
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self._login_failures: Dict[str, List[float]] = {}
|
||||
self._enroll_failures: Dict[str, List[float]] = {}
|
||||
|
||||
@staticmethod
|
||||
def get_client_ip(request: Optional[Request] = None) -> str:
|
||||
"""Extracts client IP address respecting reverse-proxy headers."""
|
||||
if not request:
|
||||
return "127.0.0.1"
|
||||
forwarded = request.headers.get("x-forwarded-for")
|
||||
if forwarded:
|
||||
return forwarded.split(",")[0].strip()
|
||||
real_ip = request.headers.get("x-real-ip")
|
||||
if real_ip:
|
||||
return real_ip.strip()
|
||||
if request.client:
|
||||
return request.client.host
|
||||
return "127.0.0.1"
|
||||
|
||||
def check_ip_rate_limit(
|
||||
self,
|
||||
ip: str,
|
||||
store: Any = None,
|
||||
max_attempts: int = 10,
|
||||
window_seconds: int = 60,
|
||||
scope: str = "login",
|
||||
db_path: Optional[str] = None,
|
||||
fallback_db: str = "logar_state.db"
|
||||
) -> bool:
|
||||
"""Checks whether the client IP has exceeded failed attempts within the sliding window."""
|
||||
now = datetime.now(timezone.utc).timestamp()
|
||||
cutoff = now - window_seconds
|
||||
target_db = db_path or fallback_db
|
||||
try:
|
||||
conn = sqlite3.connect(target_db, timeout=5.0)
|
||||
c = conn.cursor()
|
||||
c.execute("CREATE TABLE IF NOT EXISTS rate_limits (ip TEXT NOT NULL, scope TEXT NOT NULL, attempt_time REAL NOT NULL)")
|
||||
c.execute("DELETE FROM rate_limits WHERE attempt_time < ?", (cutoff,))
|
||||
c.execute("SELECT COUNT(*) FROM rate_limits WHERE ip = ? AND scope = ? AND attempt_time >= ?", (ip, scope, cutoff))
|
||||
cnt = c.fetchone()[0]
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return cnt < max_attempts
|
||||
except Exception:
|
||||
with self._lock:
|
||||
effective_store = store if isinstance(store, dict) else (self._login_failures if scope == "login" else self._enroll_failures)
|
||||
history = effective_store.get(ip, [])
|
||||
history = [ts for ts in history if now - ts < window_seconds]
|
||||
effective_store[ip] = history
|
||||
return len(history) < max_attempts
|
||||
|
||||
def record_failed_ip_attempt(
|
||||
self,
|
||||
ip: str,
|
||||
store: Any = None,
|
||||
scope: str = "login",
|
||||
db_path: Optional[str] = None,
|
||||
fallback_db: str = "logar_state.db"
|
||||
):
|
||||
"""Records a persistent failed attempt timestamp for the client IP and scope."""
|
||||
now = datetime.now(timezone.utc).timestamp()
|
||||
target_db = db_path or fallback_db
|
||||
try:
|
||||
conn = sqlite3.connect(target_db, timeout=5.0)
|
||||
c = conn.cursor()
|
||||
c.execute("CREATE TABLE IF NOT EXISTS rate_limits (ip TEXT NOT NULL, scope TEXT NOT NULL, attempt_time REAL NOT NULL)")
|
||||
c.execute("INSERT INTO rate_limits (ip, scope, attempt_time) VALUES (?, ?, ?)", (ip, scope, now))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
with self._lock:
|
||||
effective_store = store if isinstance(store, dict) else (self._login_failures if scope == "login" else self._enroll_failures)
|
||||
if ip not in effective_store:
|
||||
effective_store[ip] = []
|
||||
effective_store[ip].append(now)
|
||||
|
||||
def reset_failed_ip_attempts(
|
||||
self,
|
||||
ip: str,
|
||||
store: Any = None,
|
||||
scope: str = "login",
|
||||
db_path: Optional[str] = None,
|
||||
fallback_db: str = "logar_state.db"
|
||||
):
|
||||
"""Resets persistent failed attempt records for client IP upon successful authentication."""
|
||||
target_db = db_path or fallback_db
|
||||
try:
|
||||
conn = sqlite3.connect(target_db, timeout=5.0)
|
||||
c = conn.cursor()
|
||||
c.execute("DELETE FROM rate_limits WHERE ip = ? AND scope = ?", (ip, scope))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
with self._lock:
|
||||
effective_store = store if isinstance(store, dict) else (self._login_failures if scope == "login" else self._enroll_failures)
|
||||
if ip in effective_store:
|
||||
del effective_store[ip]
|
||||
|
||||
|
||||
class PasswordManager:
|
||||
"""PBKDF2-HMAC-SHA256 password hashing and validation."""
|
||||
|
||||
@staticmethod
|
||||
def hash_password(password: str, salt: Optional[str] = None) -> Tuple[str, str]:
|
||||
"""Hashes a password using PBKDF2-HMAC-SHA256 with 100,000 iterations."""
|
||||
if not salt:
|
||||
salt = secrets.token_hex(16)
|
||||
pwd_hash = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt.encode("utf-8"), 100000).hex()
|
||||
return pwd_hash, salt
|
||||
|
||||
@staticmethod
|
||||
def verify_password(password: str, salt: str, password_hash: str) -> bool:
|
||||
"""Verifies a password against the stored salt and PBKDF2 hash."""
|
||||
test_hash, _ = PasswordManager.hash_password(password, salt=salt)
|
||||
return hmac.compare_digest(test_hash, password_hash)
|
||||
|
||||
@staticmethod
|
||||
def verify_user_credentials(username: str, password: str, db_path: str = "logar_state.db") -> bool:
|
||||
"""Checks user credentials against SQLite users table."""
|
||||
conn = sqlite3.connect(db_path)
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT password_hash, salt FROM users WHERE username = ?", (username,))
|
||||
row = c.fetchone()
|
||||
conn.close()
|
||||
if not row:
|
||||
return False
|
||||
return PasswordManager.verify_password(password, row[1], row[0])
|
||||
|
||||
@staticmethod
|
||||
def change_user_password(username: str, new_password: str, db_path: str = "logar_state.db") -> bool:
|
||||
"""Updates a user's password."""
|
||||
pwd_hash, salt = PasswordManager.hash_password(new_password)
|
||||
conn = sqlite3.connect(db_path)
|
||||
c = conn.cursor()
|
||||
c.execute("UPDATE users SET password_hash = ?, salt = ? WHERE username = ?", (pwd_hash, salt, username))
|
||||
changed = c.rowcount > 0
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return changed
|
||||
|
||||
|
||||
class TOTPManager:
|
||||
"""RFC 6238 Two-Factor Authentication (TOTP) generator and validator."""
|
||||
|
||||
@staticmethod
|
||||
def generate_totp_secret(length: int = 20) -> str:
|
||||
"""Generates a cryptographically random Base32-encoded TOTP secret key."""
|
||||
return base64.b32encode(secrets.token_bytes(length)).decode("utf-8").replace("=", "")
|
||||
|
||||
@staticmethod
|
||||
def get_totp_token(secret: str, intervals_no: Optional[int] = None) -> str:
|
||||
"""Calculates a 6-digit TOTP code for the given counter interval according to RFC 6238."""
|
||||
if intervals_no is None:
|
||||
intervals_no = int(time.time()) // 30
|
||||
key = base64.b32decode(secret + "=" * ((8 - len(secret) % 8) % 8), casefold=True)
|
||||
msg = struct.pack(">Q", intervals_no)
|
||||
h = hmac.new(key, msg, hashlib.sha1).digest()
|
||||
o = h[19] & 15
|
||||
h_int = (struct.unpack(">I", h[o:o+4])[0] & 0x7fffffff) % 1000000
|
||||
return f"{h_int:06d}"
|
||||
|
||||
@staticmethod
|
||||
def verify_totp_code(secret: str, code: str, window: int = 1) -> bool:
|
||||
"""Verifies a 6-digit TOTP code with standard drift window (+/- 30s)."""
|
||||
if not secret or not code:
|
||||
return False
|
||||
code = str(code).strip().replace(" ", "")
|
||||
if len(code) != 6 or not code.isdigit():
|
||||
return False
|
||||
now = int(time.time())
|
||||
current_step = now // 30
|
||||
for offset in range(-window, window + 1):
|
||||
if hmac.compare_digest(TOTPManager.get_totp_token(secret, current_step + offset), code):
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_totp_uri(username: str, secret: str, issuer: str = "LOGAR Server Hub") -> str:
|
||||
"""Generates standard otpauth URI for QR code generation."""
|
||||
user_enc = urllib.parse.quote(username)
|
||||
issuer_enc = urllib.parse.quote(issuer)
|
||||
return f"otpauth://totp/{issuer_enc}:{user_enc}?secret={secret}&issuer={issuer_enc}&algorithm=SHA1&digits=6&period=30"
|
||||
|
||||
@staticmethod
|
||||
def get_user_totp_info(username: str, db_path: str = "logar_state.db") -> Dict[str, Any]:
|
||||
"""Retrieves TOTP configuration for a user from the database."""
|
||||
conn = sqlite3.connect(db_path)
|
||||
c = conn.cursor()
|
||||
try:
|
||||
c.execute("SELECT totp_secret, totp_enabled FROM users WHERE username = ?", (username,))
|
||||
row = c.fetchone()
|
||||
if row:
|
||||
return {"secret": row[0] or "", "enabled": bool(row[1])}
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
conn.close()
|
||||
return {"secret": "", "enabled": False}
|
||||
|
||||
@staticmethod
|
||||
def enable_user_totp(username: str, secret: str, db_path: str = "logar_state.db") -> bool:
|
||||
"""Enables and activates TOTP for a user in the database."""
|
||||
conn = sqlite3.connect(db_path)
|
||||
c = conn.cursor()
|
||||
try:
|
||||
c.execute("UPDATE users SET totp_secret = ?, totp_enabled = 1 WHERE username = ?", (secret, username))
|
||||
changed = c.rowcount > 0
|
||||
conn.commit()
|
||||
return changed
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@staticmethod
|
||||
def disable_user_totp(username: str, db_path: str = "logar_state.db") -> bool:
|
||||
"""Disables TOTP for a user in the database."""
|
||||
conn = sqlite3.connect(db_path)
|
||||
c = conn.cursor()
|
||||
try:
|
||||
c.execute("UPDATE users SET totp_secret = NULL, totp_enabled = 0 WHERE username = ?", (username,))
|
||||
changed = c.rowcount > 0
|
||||
conn.commit()
|
||||
return changed
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
class HermesTokenManager:
|
||||
"""Manages Bearer API tokens for Hermes AI anomaly integration."""
|
||||
|
||||
@staticmethod
|
||||
def create_hermes_token(name: str, token: Optional[str] = None, db_path: str = "logar_state.db") -> Dict[str, Any]:
|
||||
"""Creates or saves a named Hermes API token."""
|
||||
if not token:
|
||||
token = f"hermes_{secrets.token_hex(20)}"
|
||||
now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
conn = sqlite3.connect(db_path)
|
||||
c = conn.cursor()
|
||||
c.execute(
|
||||
"INSERT OR REPLACE INTO hermes_tokens (token, name, created_at, status) VALUES (?, ?, ?, 'active')",
|
||||
(token, name, now_iso)
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return {"token": token, "name": name, "created_at": now_iso, "status": "active"}
|
||||
|
||||
@staticmethod
|
||||
def list_hermes_tokens(db_path: str = "logar_state.db") -> List[Dict[str, Any]]:
|
||||
"""Lists all registered Hermes access tokens."""
|
||||
conn = sqlite3.connect(db_path)
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT token, name, created_at, last_used, status FROM hermes_tokens ORDER BY created_at DESC")
|
||||
rows = [
|
||||
{"token": r[0], "name": r[1], "created_at": r[2], "last_used": r[3] or "Never", "status": r[4]}
|
||||
for r in c.fetchall()
|
||||
]
|
||||
conn.close()
|
||||
return rows
|
||||
|
||||
@staticmethod
|
||||
def revoke_hermes_token(token: str, db_path: str = "logar_state.db") -> bool:
|
||||
"""Revokes an active Hermes token."""
|
||||
conn = sqlite3.connect(db_path)
|
||||
c = conn.cursor()
|
||||
c.execute("UPDATE hermes_tokens SET status = 'revoked' WHERE token = ?", (token,))
|
||||
changed = c.rowcount > 0
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return changed
|
||||
|
||||
@staticmethod
|
||||
def delete_hermes_token(token: str, db_path: str = "logar_state.db") -> bool:
|
||||
"""Deletes a Hermes token."""
|
||||
conn = sqlite3.connect(db_path)
|
||||
c = conn.cursor()
|
||||
c.execute("DELETE FROM hermes_tokens WHERE token = ?", (token,))
|
||||
changed = c.rowcount > 0
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return changed
|
||||
|
||||
@staticmethod
|
||||
def verify_hermes_token(token: str, db_path: str = "logar_state.db") -> bool:
|
||||
"""Verifies a Hermes token and updates last_used timestamp."""
|
||||
if not token:
|
||||
return False
|
||||
conn = sqlite3.connect(db_path)
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT token FROM hermes_tokens WHERE token = ? AND status = 'active'", (token,))
|
||||
row = c.fetchone()
|
||||
if row:
|
||||
now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
c.execute("UPDATE hermes_tokens SET last_used = ? WHERE token = ?", (now_iso, token))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return row is not None
|
||||
@@ -0,0 +1,114 @@
|
||||
"""
|
||||
LOGAR - Ingestion Queue Manager
|
||||
Provides a high-throughput thread-safe ring buffer and batch processor for incoming telemetry.
|
||||
Eliminates disk write contention by decoupling TCP frame ingress from physical database writes.
|
||||
"""
|
||||
|
||||
import time
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, Any, List, Optional, Callable
|
||||
|
||||
|
||||
class IngestQueueManager:
|
||||
"""
|
||||
High-throughput in-memory buffering and batch processor for incoming telemetry and log anomalies.
|
||||
Eliminates SQLite / SQL disk write contention by decoupling TCP frame ingress from physical DB writes.
|
||||
"""
|
||||
def __init__(self, flush_interval: float = 1.0, batch_size: int = 100, max_queue: int = 50000):
|
||||
self.flush_interval = flush_interval
|
||||
self.batch_size = batch_size
|
||||
self.max_queue = max_queue
|
||||
self._lock = threading.Lock()
|
||||
self._buffer: List[Dict[str, Any]] = []
|
||||
self._total_ingested = 0
|
||||
self._total_flushed = 0
|
||||
self._dropped_count = 0
|
||||
self._last_flush_time = "Never"
|
||||
self._avg_flush_latency_ms = 0.0
|
||||
self._flush_count = 0
|
||||
|
||||
def push(self, payload: Dict[str, Any]) -> bool:
|
||||
"""Pushes a log payload into the ring buffer."""
|
||||
with self._lock:
|
||||
if len(self._buffer) >= self.max_queue:
|
||||
self._dropped_count += 1
|
||||
return False
|
||||
self._buffer.append(payload)
|
||||
self._total_ingested += len(payload.get("logs", []))
|
||||
return True
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""Returns live statistics about queue occupancy, throughput, and flush latency."""
|
||||
with self._lock:
|
||||
return {
|
||||
"queue_size": len(self._buffer),
|
||||
"total_ingested": self._total_ingested,
|
||||
"total_flushed": self._total_flushed,
|
||||
"dropped_count": self._dropped_count,
|
||||
"last_flush_time": self._last_flush_time,
|
||||
"avg_flush_latency_ms": round(self._avg_flush_latency_ms, 2),
|
||||
"flush_count": self._flush_count,
|
||||
"flush_interval_seconds": self.flush_interval,
|
||||
"batch_size_threshold": self.batch_size
|
||||
}
|
||||
|
||||
def flush(
|
||||
self,
|
||||
db_path: Optional[str] = None,
|
||||
window_hours: int = 12,
|
||||
min_runs: int = 1,
|
||||
process_callback: Optional[Callable[..., Dict[str, Any]]] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Flushes all queued items to persistent storage."""
|
||||
with self._lock:
|
||||
if not self._buffer:
|
||||
return {"status": "noop", "flushed_runs": 0, "processed_logs": 0}
|
||||
items_to_process = list(self._buffer)
|
||||
self._buffer.clear()
|
||||
|
||||
start_time = time.time()
|
||||
target_db = db_path or "logar_state.db"
|
||||
|
||||
total_processed = 0
|
||||
promoted_total = 0
|
||||
|
||||
try:
|
||||
for payload in items_to_process:
|
||||
if process_callback:
|
||||
res = process_callback(
|
||||
payload=payload,
|
||||
db_path=target_db,
|
||||
window_hours=window_hours,
|
||||
min_runs=min_runs
|
||||
)
|
||||
else:
|
||||
# Fallback if imported directly
|
||||
from src import Server
|
||||
res = Server.process_ingested_logs(
|
||||
payload=payload,
|
||||
db_path=target_db,
|
||||
window_hours=window_hours,
|
||||
min_runs=min_runs
|
||||
)
|
||||
total_processed += res.get("processed", 0)
|
||||
promoted_total += res.get("promoted_verified", 0)
|
||||
except Exception as e:
|
||||
print(f"[!] Error during batch flush to DB: {e}")
|
||||
|
||||
elapsed_ms = (time.time() - start_time) * 1000.0
|
||||
with self._lock:
|
||||
self._total_flushed += total_processed
|
||||
self._flush_count += 1
|
||||
self._avg_flush_latency_ms = (
|
||||
(self._avg_flush_latency_ms * (self._flush_count - 1) + elapsed_ms) / self._flush_count
|
||||
)
|
||||
self._last_flush_time = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"flushed_runs": len(items_to_process),
|
||||
"processed_logs": total_processed,
|
||||
"promoted_verified": promoted_total,
|
||||
"flush_latency_ms": round(elapsed_ms, 2)
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
"""
|
||||
LOGAR - Licensing & Sub-CA Introspection Manager
|
||||
Provides modular classes for PKI X.509 extension parsing, capability gating,
|
||||
Subordinate CA seat tracking, and asynchronous license reporting.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import socket
|
||||
import sqlite3
|
||||
import threading
|
||||
from typing import Optional, Dict, Any, List, Tuple
|
||||
|
||||
try:
|
||||
from src import server_enrollment as enrollment
|
||||
except ImportError:
|
||||
import server_enrollment as enrollment
|
||||
|
||||
|
||||
class LicenseManager:
|
||||
"""Manages PKI licensing introspection, feature unlocking, and upstream seat syncing."""
|
||||
|
||||
DEFAULT_DB_FILE = "logar_state.db"
|
||||
|
||||
@staticmethod
|
||||
def export_license_request_lic(config: Dict[str, Any]) -> Tuple[Dict[str, Any], str]:
|
||||
"""Generates CA CSR and packages into exportable .lic dictionary and JSON string."""
|
||||
cert_dir = config.get("cert_dir", "certs")
|
||||
lic_server = config.get("license_server_url", "https://logar.eibl.tech")
|
||||
hostname = socket.gethostname()
|
||||
seats = config.get("max_seats", 10)
|
||||
fp = config.get("server_fingerprint", "")
|
||||
return enrollment.create_license_request_lic(
|
||||
cert_dir=cert_dir,
|
||||
license_server=lic_server,
|
||||
server_fingerprint=fp,
|
||||
hostname=hostname,
|
||||
requested_seats=seats
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_active_licensed_features(db_path: Optional[str] = None, server_state: Optional[Dict[str, Any]] = None) -> List[str]:
|
||||
"""
|
||||
Returns the list of all active enterprise capability identifiers
|
||||
authorized either via verified X.509 certificate extensions or the persistent database license config.
|
||||
"""
|
||||
features = set()
|
||||
state = server_state or {}
|
||||
cert_dir = state.get("config", {}).get("cert_dir", "certs")
|
||||
upstream_pem = None
|
||||
upstream_file = os.path.join(cert_dir, "upstream_ca.crt")
|
||||
if os.path.exists(upstream_file):
|
||||
try:
|
||||
with open(upstream_file, "r", encoding="utf-8") as f:
|
||||
upstream_pem = f.read()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 1. Check in-memory / active Subordinate CA certificate
|
||||
ca_pem = state.get("ca_cert_pem")
|
||||
if not ca_pem:
|
||||
ca_file = os.path.join(cert_dir, "ca.crt")
|
||||
if os.path.exists(ca_file):
|
||||
try:
|
||||
with open(ca_file, "r", encoding="utf-8") as f:
|
||||
ca_pem = f.read()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
cert_is_verified = False
|
||||
if ca_pem:
|
||||
try:
|
||||
info = enrollment.extract_license_info_from_cert(ca_pem, upstream_ca_pem=upstream_pem)
|
||||
cert_is_verified = info.get("signature_verified", False) or not info.get("is_self_signed", False)
|
||||
for feat in info.get("features", []):
|
||||
if feat:
|
||||
features.add(str(feat).strip())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2. Check license_config table in database
|
||||
target_db = db_path or state.get("config", {}).get("db_path", LicenseManager.DEFAULT_DB_FILE)
|
||||
if target_db and os.path.exists(target_db):
|
||||
try:
|
||||
conn = sqlite3.connect(target_db, timeout=3.0)
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT features FROM license_config WHERE id = 1")
|
||||
row = c.fetchone()
|
||||
conn.close()
|
||||
if row and row[0]:
|
||||
feat_raw = row[0]
|
||||
if isinstance(feat_raw, str):
|
||||
if feat_raw.startswith("["):
|
||||
feat_list = json.loads(feat_raw)
|
||||
else:
|
||||
feat_list = [f.strip() for f in feat_raw.split(",") if f.strip()]
|
||||
for f_item in feat_list:
|
||||
if f_item:
|
||||
features.add(str(f_item).strip())
|
||||
elif isinstance(feat_raw, list):
|
||||
for f_item in feat_raw:
|
||||
if f_item:
|
||||
features.add(str(f_item).strip())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return sorted(list(features))
|
||||
|
||||
@staticmethod
|
||||
def is_feature_licensed(feature: str, db_path: Optional[str] = None, server_state: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""Checks whether a specific enterprise capability (e.g. 'enterprise_db') is authorized."""
|
||||
if not feature:
|
||||
return False
|
||||
return feature in LicenseManager.get_active_licensed_features(db_path=db_path, server_state=server_state)
|
||||
|
||||
@staticmethod
|
||||
def get_license_summary(server_state: Optional[Dict[str, Any]] = None, db_path: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Inspects certs/ca.crt, database quotas, verifies cryptographic signatures, and returns comprehensive licensing status."""
|
||||
state = server_state or {}
|
||||
config = state.get("config", {})
|
||||
cert_dir = config.get("cert_dir", "certs")
|
||||
ca_path = os.path.join(cert_dir, "ca.crt")
|
||||
upstream_path = os.path.join(cert_dir, "upstream_ca.crt")
|
||||
target_db = db_path or config.get("db_path", LicenseManager.DEFAULT_DB_FILE)
|
||||
|
||||
upstream_pem = None
|
||||
if os.path.exists(upstream_path):
|
||||
try:
|
||||
with open(upstream_path, "r", encoding="utf-8") as f:
|
||||
upstream_pem = f.read().strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
db_max_seats = config.get("max_seats", 0)
|
||||
license_status_db = "unlicensed"
|
||||
try:
|
||||
conn = sqlite3.connect(target_db)
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT max_seats, license_status FROM license_config WHERE id = 1")
|
||||
row = c.fetchone()
|
||||
if row:
|
||||
if row[0] is not None:
|
||||
db_max_seats = row[0]
|
||||
if len(row) > 1 and row[1]:
|
||||
license_status_db = row[1]
|
||||
c.execute("SELECT COUNT(*) FROM clients WHERE status = 'active' AND cert_fingerprint != 'LOCAL-HOST-BUILTIN'")
|
||||
active_seats = c.fetchone()[0]
|
||||
c.execute("SELECT COUNT(*) FROM clients WHERE status = 'revoked' AND cert_fingerprint != 'LOCAL-HOST-BUILTIN'")
|
||||
revoked_seats = c.fetchone()[0]
|
||||
conn.close()
|
||||
except Exception:
|
||||
active_seats = 0
|
||||
revoked_seats = 0
|
||||
|
||||
csr_path = os.path.join(cert_dir, "ca.csr")
|
||||
csr_exists = os.path.exists(csr_path)
|
||||
|
||||
cert_seats = None
|
||||
if os.path.exists(ca_path):
|
||||
try:
|
||||
with open(ca_path, "r", encoding="utf-8") as f:
|
||||
ca_pem = f.read()
|
||||
info = enrollment.extract_license_info_from_cert(ca_pem, upstream_ca_pem=upstream_pem)
|
||||
if info.get("seats") is not None:
|
||||
cert_seats = info["seats"]
|
||||
except Exception:
|
||||
info = {"valid": False}
|
||||
else:
|
||||
info = {"valid": False, "is_self_signed": False, "is_expired": False, "days_remaining": 0}
|
||||
|
||||
# Anti-tampering reconciliation: If database max_seats diverged from certified quota, enforce certified quota and update DB
|
||||
if cert_seats is not None:
|
||||
max_seats = cert_seats
|
||||
if db_max_seats != cert_seats and os.path.exists(target_db):
|
||||
try:
|
||||
conn = sqlite3.connect(target_db)
|
||||
conn.execute("UPDATE license_config SET max_seats = ? WHERE id = 1", (cert_seats,))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
max_seats = db_max_seats
|
||||
|
||||
license_id = info.get("license_id") or "UNLICENSED"
|
||||
issuer_url = info.get("issuer_url") or config.get("license_server_url", "https://logar.eibl.tech")
|
||||
|
||||
if info.get("is_expired"):
|
||||
status_label = "EXPIRED"
|
||||
elif info.get("seats") and not info.get("is_self_signed"):
|
||||
status_label = "ACTIVE"
|
||||
elif max_seats > 0 and license_status_db == "active":
|
||||
status_label = "ACTIVE"
|
||||
elif info.get("is_self_signed"):
|
||||
status_label = "SELF-SIGNED" if max_seats > 0 else "UNLICENSED"
|
||||
else:
|
||||
status_label = "UNLICENSED" if max_seats <= 0 else "ACTIVE"
|
||||
|
||||
pct = int((active_seats / max_seats * 100)) if max_seats > 0 else 0
|
||||
return {
|
||||
"status": status_label,
|
||||
"active_seats": active_seats,
|
||||
"active_seats_taken": active_seats,
|
||||
"revoked_seats": revoked_seats,
|
||||
"max_seats": max_seats,
|
||||
"available_seats": max(0, max_seats - active_seats),
|
||||
"percentage": min(100, pct),
|
||||
"license_id": license_id,
|
||||
"issuer_url": issuer_url,
|
||||
"issuer_cn": info.get("issuer_cn", "N/A"),
|
||||
"subject_cn": info.get("subject_cn", "N/A"),
|
||||
"is_self_signed": info.get("is_self_signed", False),
|
||||
"signature_verified": info.get("signature_verified", False),
|
||||
"not_valid_before": info.get("not_valid_before", "N/A"),
|
||||
"not_valid_after": info.get("not_valid_after", "N/A"),
|
||||
"days_remaining": info.get("days_remaining", 0),
|
||||
"fingerprint": info.get("fingerprint", "N/A"),
|
||||
"csr_exists": csr_exists,
|
||||
"server_fingerprint": config.get("server_fingerprint", "N/A"),
|
||||
"features": LicenseManager.get_active_licensed_features(db_path=target_db, server_state=state)
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def report_seat_quota_to_authority_sync(server_state: Optional[Dict[str, Any]] = None):
|
||||
"""Background helper to push active seat count to upstream authority without blocking."""
|
||||
try:
|
||||
state = server_state or {}
|
||||
config = state.get("config", {})
|
||||
cert_dir = config.get("cert_dir", "certs")
|
||||
db_path = config.get("db_path", LicenseManager.DEFAULT_DB_FILE)
|
||||
lic_server = config.get("license_server_url", "https://logar.eibl.tech")
|
||||
if not os.path.exists(os.path.join(cert_dir, "ca.crt")) or not os.path.exists(os.path.join(cert_dir, "ca.key")):
|
||||
return
|
||||
|
||||
active_seats = 0
|
||||
if os.path.exists(db_path):
|
||||
try:
|
||||
conn = sqlite3.connect(db_path)
|
||||
row = conn.execute("SELECT COUNT(*) FROM clients WHERE status = 'active' AND cert_fingerprint != 'LOCAL-HOST-BUILTIN'").fetchone()
|
||||
if row:
|
||||
active_seats = row[0]
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
enrollment.renew_ca_cert_mtls(
|
||||
cert_dir=cert_dir,
|
||||
license_server_url=lic_server,
|
||||
seats_taken=active_seats,
|
||||
active_seats=active_seats,
|
||||
hostname=config.get("server_name", ""),
|
||||
server_fingerprint=config.get("server_fingerprint", "")
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def trigger_async_seat_report(server_state: Optional[Dict[str, Any]] = None):
|
||||
"""Fires a non-blocking background thread to sync seat usage with upstream authority."""
|
||||
threading.Thread(target=LicenseManager.report_seat_quota_to_authority_sync, args=(server_state,), daemon=True).start()
|
||||
@@ -0,0 +1,141 @@
|
||||
"""
|
||||
LOGAR - Dynamic PKI & mTLS Context Manager
|
||||
Provides modular classes for TLS 1.3 socket contexts, in-flight certificate rotation,
|
||||
and automated Sub-CA renewal.
|
||||
"""
|
||||
|
||||
import os
|
||||
import ssl
|
||||
import sqlite3
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
try:
|
||||
from src import server_enrollment as enrollment
|
||||
except ImportError:
|
||||
import server_enrollment as enrollment
|
||||
|
||||
|
||||
class PKIManager:
|
||||
"""Manages server TLS certificates, mTLS SSL contexts, and hot certificate reloading."""
|
||||
|
||||
@staticmethod
|
||||
def init_mtls_server_context(cert_dir: str = "certs") -> ssl.SSLContext:
|
||||
"""Initializes TLS 1.3 server SSLContext with client certificate requirement (mTLS)."""
|
||||
ca_file = os.path.join(cert_dir, "ca.crt")
|
||||
srv_cert = os.path.join(cert_dir, "server.crt")
|
||||
srv_key = os.path.join(cert_dir, "server.key")
|
||||
|
||||
ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
|
||||
ctx.load_cert_chain(certfile=srv_cert, keyfile=srv_key)
|
||||
ctx.load_verify_locations(cafile=ca_file)
|
||||
upstream_file = os.path.join(cert_dir, "upstream_ca.crt")
|
||||
if os.path.exists(upstream_file):
|
||||
try:
|
||||
ctx.load_verify_locations(cafile=upstream_file)
|
||||
except Exception:
|
||||
pass
|
||||
ctx.verify_mode = ssl.CERT_REQUIRED
|
||||
ctx.minimum_version = ssl.TLSVersion.TLSv1_3
|
||||
return ctx
|
||||
|
||||
@staticmethod
|
||||
def reload_mtls_context(ssl_ctx: ssl.SSLContext, cert_dir: str = "certs"):
|
||||
"""
|
||||
Dynamically reloads server certificate chain and Root CA in an active SSLContext.
|
||||
Allows in-flight TLS certificate rotation without dropping the listening socket.
|
||||
"""
|
||||
ca_file = os.path.join(cert_dir, "ca.crt")
|
||||
srv_cert = os.path.join(cert_dir, "server.crt")
|
||||
srv_key = os.path.join(cert_dir, "server.key")
|
||||
|
||||
ssl_ctx.load_cert_chain(certfile=srv_cert, keyfile=srv_key)
|
||||
ssl_ctx.load_verify_locations(cafile=ca_file)
|
||||
upstream_file = os.path.join(cert_dir, "upstream_ca.crt")
|
||||
if os.path.exists(upstream_file):
|
||||
try:
|
||||
ssl_ctx.load_verify_locations(cafile=upstream_file)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def check_and_rotate_server_certs(
|
||||
cert_dir: str = "certs",
|
||||
hostnames: Optional[List[str]] = None,
|
||||
threshold_days: int = 30,
|
||||
server_state: Optional[Dict[str, Any]] = None
|
||||
) -> bool:
|
||||
"""
|
||||
Checks if Root/Subordinate CA or server TLS certificate are expiring within threshold_days.
|
||||
If a licensed Subordinate CA is expiring, automatically connects over mTLS to logar.eibl.tech
|
||||
to request an in-flight renewal, then reloads the active SSLContext without socket interruption.
|
||||
Returns True if renewed/reloaded, False otherwise.
|
||||
"""
|
||||
state = server_state or {}
|
||||
config = state.get("config", {})
|
||||
license_server = config.get("license_server_url", "https://logar.eibl.tech")
|
||||
ca_cert_path = os.path.join(cert_dir, "ca.crt")
|
||||
|
||||
# 1. Check if CA certificate is expiring and attempt upstream mTLS renewal
|
||||
if os.path.exists(ca_cert_path):
|
||||
try:
|
||||
with open(ca_cert_path, "r", encoding="utf-8") as f:
|
||||
ca_pem = f.read()
|
||||
if enrollment.is_cert_expiring_soon(ca_pem, threshold_days=threshold_days):
|
||||
info = enrollment.extract_license_info_from_cert(ca_pem)
|
||||
if not info.get("is_self_signed", True) or info.get("license_id"):
|
||||
print("[*] Subordinate CA expiring soon. Initiating automated mTLS renewal with logar.eibl.tech...")
|
||||
success, msg, new_pem = enrollment.renew_ca_cert_mtls(
|
||||
cert_dir=cert_dir,
|
||||
license_server_url=license_server
|
||||
)
|
||||
if success and new_pem:
|
||||
print(f"[+] Automated mTLS renewal successful: {msg}")
|
||||
new_info = enrollment.extract_license_info_from_cert(new_pem)
|
||||
if new_info.get("seats"):
|
||||
config["max_seats"] = new_info["seats"]
|
||||
db_path = config.get("db_path", "logar_state.db")
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.execute("UPDATE license_config SET max_seats = ? WHERE id = 1", (new_info["seats"],))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
ca_cert, ca_key, ca_pem, ca_key_pem = enrollment.generate_ca_if_needed(cert_dir=cert_dir, force_renew=False)
|
||||
state["ca_cert"] = ca_cert
|
||||
state["ca_key"] = ca_key
|
||||
state["ca_cert_pem"] = ca_pem
|
||||
enrollment.generate_server_cert_if_needed(ca_cert, ca_key, hostnames=hostnames, cert_dir=cert_dir, force_renew=True)
|
||||
ssl_ctx = state.get("ssl_ctx")
|
||||
if ssl_ctx is not None:
|
||||
try:
|
||||
PKIManager.reload_mtls_context(ssl_ctx, cert_dir=cert_dir)
|
||||
print("[+] In-flight mTLS SSLContext successfully reloaded with renewed CA.")
|
||||
except Exception as e:
|
||||
print(f"[!] Failed to reload in-flight SSLContext: {e}")
|
||||
return True
|
||||
else:
|
||||
print(f"[!] Automated mTLS renewal attempt failed: {msg}")
|
||||
except Exception as e:
|
||||
print(f"[!] Error evaluating CA certificate for mTLS renewal: {e}")
|
||||
|
||||
# 2. Local self-healing / regeneration check
|
||||
ca_renewed, srv_renewed = enrollment.check_and_renew_hub_pki(
|
||||
cert_dir=cert_dir,
|
||||
hostnames=hostnames,
|
||||
threshold_days=threshold_days
|
||||
)
|
||||
if ca_renewed or srv_renewed:
|
||||
print(f"[!] Server Hub PKI certificates renewed (CA renewed: {ca_renewed}, Server cert renewed: {srv_renewed}).")
|
||||
ca_cert, ca_key, ca_pem, ca_key_pem = enrollment.generate_ca_if_needed(cert_dir=cert_dir, force_renew=False)
|
||||
state["ca_cert"] = ca_cert
|
||||
state["ca_key"] = ca_key
|
||||
state["ca_cert_pem"] = ca_pem
|
||||
|
||||
ssl_ctx = state.get("ssl_ctx")
|
||||
if ssl_ctx is not None:
|
||||
try:
|
||||
PKIManager.reload_mtls_context(ssl_ctx, cert_dir=cert_dir)
|
||||
print("[+] In-flight mTLS SSLContext successfully reloaded with updated certificates.")
|
||||
except Exception as e:
|
||||
print(f"[!] Failed to reload in-flight SSLContext: {e}")
|
||||
return True
|
||||
return False
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,101 @@
|
||||
"""
|
||||
LOGAR - Server Pydantic Models
|
||||
Provides structured request and response models for the LOGAR Server Hub API.
|
||||
"""
|
||||
|
||||
from typing import Optional, Any, Dict, List, Union
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class ClientEnrollRequest(BaseModel):
|
||||
client_id: str
|
||||
hostname: str
|
||||
os: str
|
||||
enrollment_secret: str
|
||||
version: Optional[str] = None
|
||||
|
||||
|
||||
class ClientUpdateRequest(BaseModel):
|
||||
client_id: str
|
||||
os: str
|
||||
current_version: Optional[str] = None
|
||||
arch: Optional[str] = "x86_64"
|
||||
|
||||
|
||||
class ServerUpdateCheckRequest(BaseModel):
|
||||
force: bool = False
|
||||
|
||||
|
||||
class ClientActionRequest(BaseModel):
|
||||
client_id: str
|
||||
|
||||
|
||||
class LicenseImportRequest(BaseModel):
|
||||
ca_cert: Optional[str] = None
|
||||
upstream_ca: Optional[str] = None
|
||||
master_root_ca: Optional[str] = None
|
||||
lic: Optional[str] = None
|
||||
data: Optional[str] = None
|
||||
format: Optional[str] = None
|
||||
license_id: Optional[str] = None
|
||||
seats: Optional[int] = None
|
||||
features: Optional[Any] = None
|
||||
communication_cert: Optional[str] = None
|
||||
communication_key: Optional[str] = None
|
||||
license_authority: Optional[str] = None
|
||||
|
||||
class Config:
|
||||
extra = "allow"
|
||||
|
||||
|
||||
class HermesTokenCreateRequest(BaseModel):
|
||||
name: str
|
||||
|
||||
|
||||
class HermesTokenRevokeRequest(BaseModel):
|
||||
token: str
|
||||
|
||||
|
||||
class ClientProfileExportRequest(BaseModel):
|
||||
server_host: Optional[str] = None
|
||||
server_port: Optional[int] = None
|
||||
client_id: Optional[str] = None
|
||||
|
||||
|
||||
class ChangePasswordRequest(BaseModel):
|
||||
old_password: str
|
||||
new_password: str
|
||||
|
||||
|
||||
class DbConnectionTestRequest(BaseModel):
|
||||
host: str = "localhost"
|
||||
port: int = 5432
|
||||
database: str = "logar"
|
||||
user: str = "postgres"
|
||||
password: str = ""
|
||||
sslmode: str = "prefer"
|
||||
connect_timeout: int = 5
|
||||
pool_size: int = 10
|
||||
|
||||
|
||||
class DbSwitchBackendRequest(BaseModel):
|
||||
db_type: str = "sqlite" # "sqlite" | "postgres"
|
||||
host: Optional[str] = "localhost"
|
||||
port: Optional[int] = 5432
|
||||
database: Optional[str] = "logar"
|
||||
user: Optional[str] = "postgres"
|
||||
password: Optional[str] = ""
|
||||
sslmode: Optional[str] = "prefer"
|
||||
connect_timeout: Optional[int] = 5
|
||||
pool_size: Optional[int] = 10
|
||||
db_path: Optional[str] = None
|
||||
|
||||
|
||||
class DbFullConfigRequest(BaseModel):
|
||||
storage_engine: str = "sqlite" # "sqlite" | "postgres"
|
||||
cache_mode: str = "direct" # "direct" | "buffered"
|
||||
flush_interval: float = 2.0
|
||||
batch_size: int = 100
|
||||
max_queue: int = 50000
|
||||
encrypt_at_rest: bool = False
|
||||
pg_config: Optional[DbConnectionTestRequest] = None
|
||||
@@ -0,0 +1,641 @@
|
||||
"""
|
||||
LOGAR - Multi-Backend Storage Engine & Data-at-Rest Crypto
|
||||
Provides modular classes for Database schema management, SQLite WAL persistence,
|
||||
PostgreSQL enterprise backend, and OpenPGP / AES-256-GCM data encryption at rest.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import sqlite3
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, Dict, Any, List, Tuple, Callable
|
||||
import pgpy
|
||||
from pgpy.constants import (
|
||||
PubKeyAlgorithm,
|
||||
KeyFlags,
|
||||
HashAlgorithm,
|
||||
SymmetricKeyAlgorithm,
|
||||
CompressionAlgorithm
|
||||
)
|
||||
|
||||
from src.auth_manager import PasswordManager
|
||||
|
||||
|
||||
class StorageCryptoManager:
|
||||
"""Manages OpenPGP keypairs and AES-256-GCM zero-knowledge data-at-rest encryption."""
|
||||
|
||||
@staticmethod
|
||||
def generate_server_keypair(server_name: str) -> Tuple[str, str, str]:
|
||||
"""Generates an OpenPGP RSA 2048 key with encryption capability."""
|
||||
key = pgpy.PGPKey.new(PubKeyAlgorithm.RSAEncryptOrSign, 2048)
|
||||
uid = pgpy.PGPUID.new(server_name)
|
||||
key.add_uid(
|
||||
uid,
|
||||
usage={KeyFlags.EncryptCommunications, KeyFlags.EncryptStorage},
|
||||
hashes=[HashAlgorithm.SHA256, HashAlgorithm.SHA512],
|
||||
ciphers=[SymmetricKeyAlgorithm.AES256, SymmetricKeyAlgorithm.AES128],
|
||||
compression=[
|
||||
CompressionAlgorithm.ZIP,
|
||||
CompressionAlgorithm.ZLIB,
|
||||
CompressionAlgorithm.BZ2,
|
||||
CompressionAlgorithm.Uncompressed
|
||||
]
|
||||
)
|
||||
private_key_armored = str(key)
|
||||
public_key_armored = str(key.pubkey)
|
||||
fingerprint = str(key.pubkey.fingerprint)
|
||||
return private_key_armored, public_key_armored, fingerprint
|
||||
|
||||
@staticmethod
|
||||
def get_server_public_key(server_state: Optional[Dict[str, Any]] = None) -> Optional[pgpy.PGPKey]:
|
||||
"""Retrieves or parses the server's OpenPGP public key for encryption operations."""
|
||||
state = server_state or {}
|
||||
pub_obj = state.get("public_key_obj")
|
||||
if pub_obj:
|
||||
return pub_obj
|
||||
priv_obj = state.get("private_key_obj")
|
||||
if priv_obj and hasattr(priv_obj, "pubkey"):
|
||||
state["public_key_obj"] = priv_obj.pubkey
|
||||
return priv_obj.pubkey
|
||||
pub_blob = state.get("config", {}).get("public_key")
|
||||
if pub_blob:
|
||||
try:
|
||||
key, _ = pgpy.PGPKey.from_blob(pub_blob)
|
||||
state["public_key_obj"] = key
|
||||
return key
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def encrypt_field_for_storage(text: str, force: bool = False, server_state: Optional[Dict[str, Any]] = None, is_unlocked: bool = False) -> str:
|
||||
"""Encrypts a text payload using the server's OpenPGP public key if encrypt_at_rest is enabled."""
|
||||
if not text or not isinstance(text, str):
|
||||
return text
|
||||
state = server_state or {}
|
||||
config = state.get("config", {})
|
||||
should_encrypt = force or (is_unlocked and bool(config.get("encrypt_at_rest", False)))
|
||||
if not should_encrypt:
|
||||
return text
|
||||
|
||||
if text.startswith("-----BEGIN PGP MESSAGE-----"):
|
||||
return text
|
||||
|
||||
pub_key = StorageCryptoManager.get_server_public_key(server_state=state)
|
||||
if not pub_key:
|
||||
return text
|
||||
|
||||
try:
|
||||
msg = pgpy.PGPMessage.new(text)
|
||||
encrypted = pub_key.encrypt(msg)
|
||||
return str(encrypted)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Failed to encrypt field at rest: {e}")
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def decrypt_field_from_storage(text: str, server_state: Optional[Dict[str, Any]] = None) -> str:
|
||||
"""
|
||||
Decrypts an OpenPGP armored message string using the server's private key.
|
||||
Returns cleartext for authorized display and API consumption.
|
||||
"""
|
||||
if not text or not isinstance(text, str):
|
||||
return text
|
||||
if not text.startswith("-----BEGIN PGP MESSAGE-----"):
|
||||
return text
|
||||
|
||||
state = server_state or {}
|
||||
priv_key = state.get("private_key_obj")
|
||||
if not priv_key:
|
||||
priv_blob = state.get("config", {}).get("private_key")
|
||||
if priv_blob:
|
||||
try:
|
||||
priv_key, _ = pgpy.PGPKey.from_blob(priv_blob)
|
||||
state["private_key_obj"] = priv_key
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not priv_key:
|
||||
return text
|
||||
|
||||
try:
|
||||
pgp_msg = pgpy.PGPMessage.from_blob(text)
|
||||
decrypted = priv_key.decrypt(pgp_msg)
|
||||
return str(decrypted.message)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Failed to decrypt field from storage: {e}")
|
||||
return text
|
||||
|
||||
|
||||
class StorageEngine:
|
||||
"""Manages database schema initialization, migrations, multi-engine switching, and telemetry stats."""
|
||||
|
||||
DEFAULT_DB_FILE = "logar_state.db"
|
||||
DEFAULT_ADMIN_USER = os.environ.get("LOGAR_ADMIN_USER", "admin")
|
||||
DEFAULT_ADMIN_PASS = os.environ.get("LOGAR_ADMIN_PASS", "admin_logar_2026!")
|
||||
|
||||
@classmethod
|
||||
def init_db(cls, db_path: str, enrollment_secret: Optional[str] = None, max_seats: int = 0) -> str:
|
||||
"""Initializes the SQLite schema for multi-run temporal tracking, client tracking, users, and license quota."""
|
||||
def _execute_schema(target_path: str):
|
||||
target_dir = os.path.dirname(os.path.abspath(target_path))
|
||||
if target_dir:
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
conn = sqlite3.connect(target_path, timeout=10.0)
|
||||
try:
|
||||
conn.execute("PRAGMA journal_mode=WAL;")
|
||||
conn.execute("PRAGMA busy_timeout=5000;")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 1. Active Issues
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS active_issues (
|
||||
fingerprint TEXT PRIMARY KEY,
|
||||
site_name TEXT,
|
||||
server TEXT,
|
||||
signature TEXT,
|
||||
severity TEXT,
|
||||
message TEXT,
|
||||
os_type TEXT,
|
||||
first_seen TEXT,
|
||||
last_seen TEXT,
|
||||
run_count INTEGER,
|
||||
status TEXT,
|
||||
last_run_id TEXT
|
||||
)
|
||||
""")
|
||||
|
||||
# 2. Ingest Runs
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ingest_runs (
|
||||
run_id TEXT PRIMARY KEY,
|
||||
site_name TEXT,
|
||||
server TEXT,
|
||||
timestamp TEXT,
|
||||
log_count INTEGER
|
||||
)
|
||||
""")
|
||||
|
||||
# 3. License Config
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS license_config (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
max_seats INTEGER NOT NULL DEFAULT 0,
|
||||
enrollment_secret TEXT NOT NULL,
|
||||
license_id TEXT,
|
||||
license_server_url TEXT,
|
||||
issued_at TEXT,
|
||||
expires_at TEXT,
|
||||
license_status TEXT DEFAULT 'unlicensed'
|
||||
)
|
||||
""")
|
||||
|
||||
def _ensure_columns(tbl: str, cols: List[Tuple[str, str]]):
|
||||
try:
|
||||
existing = {row[1] for row in conn.execute(f"PRAGMA table_info({tbl})").fetchall()}
|
||||
for col, col_def in cols:
|
||||
if col not in existing:
|
||||
conn.execute(f"ALTER TABLE {tbl} ADD COLUMN {col} {col_def}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Migrations for license_config
|
||||
_ensure_columns("license_config", [
|
||||
("license_id", "TEXT"),
|
||||
("license_server_url", "TEXT"),
|
||||
("issued_at", "TEXT"),
|
||||
("expires_at", "TEXT"),
|
||||
("license_status", "TEXT DEFAULT 'unlicensed'"),
|
||||
("features", "TEXT DEFAULT '[]'")
|
||||
])
|
||||
|
||||
# 4. Clients Table
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS clients (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
hostname TEXT NOT NULL,
|
||||
os_type TEXT NOT NULL,
|
||||
cert_fingerprint TEXT NOT NULL,
|
||||
status TEXT DEFAULT 'active',
|
||||
first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
# Migrations for clients table
|
||||
_ensure_columns("clients", [
|
||||
("metrics_json", "TEXT DEFAULT '{}'"),
|
||||
("cpu_percent", "REAL DEFAULT 0.0"),
|
||||
("memory_percent", "REAL DEFAULT 0.0"),
|
||||
("disk_percent_free", "REAL DEFAULT 0.0"),
|
||||
("client_version", "TEXT DEFAULT ''"),
|
||||
("update_status", "TEXT DEFAULT 'up_to_date'"),
|
||||
("last_update_check", "TEXT")
|
||||
])
|
||||
|
||||
# 5. Users Table
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
username TEXT PRIMARY KEY,
|
||||
password_hash TEXT NOT NULL,
|
||||
salt TEXT NOT NULL,
|
||||
role TEXT NOT NULL DEFAULT 'admin',
|
||||
created_at TEXT NOT NULL,
|
||||
totp_secret TEXT,
|
||||
totp_enabled INTEGER DEFAULT 0
|
||||
)
|
||||
""")
|
||||
|
||||
# Migrations for users table (2FA)
|
||||
_ensure_columns("users", [
|
||||
("totp_secret", "TEXT"),
|
||||
("totp_enabled", "INTEGER DEFAULT 0")
|
||||
])
|
||||
|
||||
# Ensure default admin user exists
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT COUNT(*) FROM users")
|
||||
if c.fetchone()[0] == 0:
|
||||
now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
pwd_hash, salt = PasswordManager.hash_password(cls.DEFAULT_ADMIN_PASS)
|
||||
conn.execute(
|
||||
"INSERT INTO users (username, password_hash, salt, role, created_at) VALUES (?, ?, ?, 'admin', ?)",
|
||||
(cls.DEFAULT_ADMIN_USER, pwd_hash, salt, now_iso)
|
||||
)
|
||||
|
||||
# 6. Hermes Tokens Table
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS hermes_tokens (
|
||||
token TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
last_used TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'active'
|
||||
)
|
||||
""")
|
||||
|
||||
# Auto-initialize initial Hermes token if table is empty
|
||||
c.execute("SELECT COUNT(*) FROM hermes_tokens")
|
||||
if c.fetchone()[0] == 0:
|
||||
initial_token = f"hermes_live_{secrets.token_hex(20)}"
|
||||
now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
conn.execute(
|
||||
"INSERT INTO hermes_tokens (token, name, created_at, status) VALUES (?, 'Default Hermes Agent', ?, 'active')",
|
||||
(initial_token, now_iso)
|
||||
)
|
||||
|
||||
# 7. Rate Limits & Server Meta Tables
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS rate_limits (
|
||||
ip TEXT NOT NULL,
|
||||
scope TEXT NOT NULL,
|
||||
attempt_time REAL NOT NULL
|
||||
)
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_rate_limits_ip_scope ON rate_limits (ip, scope, attempt_time)")
|
||||
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS server_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
)
|
||||
""")
|
||||
|
||||
status = "active" if max_seats > 0 else "unlicensed"
|
||||
conn.execute("""
|
||||
INSERT OR IGNORE INTO license_config (id, max_seats, enrollment_secret, license_status, features)
|
||||
VALUES (1, ?, ?, ?, '[]')
|
||||
""", (max_seats, enrollment_secret or "", status))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
try:
|
||||
_execute_schema(db_path)
|
||||
return db_path
|
||||
except (sqlite3.OperationalError, PermissionError, OSError) as e:
|
||||
if "readonly" in str(e).lower() or "permission" in str(e).lower() or "locked" in str(e).lower() or isinstance(e, PermissionError):
|
||||
local_app = os.environ.get("LOCALAPPDATA") or os.path.expanduser(r"~\AppData\Local")
|
||||
fallback_dir = os.path.join(local_app, "LOGAR-Server")
|
||||
os.makedirs(fallback_dir, exist_ok=True)
|
||||
fallback_path = os.path.join(fallback_dir, "logar_state.db")
|
||||
if fallback_path != db_path:
|
||||
print(f"[!] Warning: Database path '{db_path}' is not writable ({e}). Falling back to '{fallback_path}'...")
|
||||
_execute_schema(fallback_path)
|
||||
return fallback_path
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def init_postgres_db(pg_config: Dict[str, Any]) -> Tuple[bool, str]:
|
||||
"""Initializes table schema on a PostgreSQL database instance."""
|
||||
try:
|
||||
try:
|
||||
import psycopg2
|
||||
except ImportError:
|
||||
try:
|
||||
import psycopg as psycopg2
|
||||
except ImportError:
|
||||
return False, "PostgreSQL driver not installed on server (install psycopg2-binary or psycopg)"
|
||||
|
||||
host = pg_config.get("host", "localhost")
|
||||
port = int(pg_config.get("port", 5432))
|
||||
database = pg_config.get("database", "logar")
|
||||
user = pg_config.get("user", "postgres")
|
||||
password = pg_config.get("password", "")
|
||||
sslmode = pg_config.get("sslmode", "prefer")
|
||||
timeout = int(pg_config.get("connect_timeout", 5))
|
||||
|
||||
conn = psycopg2.connect(
|
||||
host=host, port=port, dbname=database, user=user, password=password, sslmode=sslmode, connect_timeout=timeout
|
||||
)
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS active_issues (
|
||||
fingerprint VARCHAR(255) PRIMARY KEY,
|
||||
site_name VARCHAR(255),
|
||||
server VARCHAR(255),
|
||||
signature TEXT,
|
||||
severity VARCHAR(50),
|
||||
message TEXT,
|
||||
os_type VARCHAR(50),
|
||||
first_seen VARCHAR(50),
|
||||
last_seen VARCHAR(50),
|
||||
run_count INTEGER,
|
||||
status VARCHAR(50),
|
||||
last_run_id VARCHAR(100)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ingest_runs (
|
||||
run_id VARCHAR(100) PRIMARY KEY,
|
||||
site_name VARCHAR(255),
|
||||
server VARCHAR(255),
|
||||
timestamp VARCHAR(50),
|
||||
log_count INTEGER
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS license_config (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
max_seats INTEGER NOT NULL DEFAULT 0,
|
||||
enrollment_secret TEXT NOT NULL,
|
||||
license_id VARCHAR(100),
|
||||
license_server_url VARCHAR(255),
|
||||
issued_at VARCHAR(50),
|
||||
expires_at VARCHAR(50),
|
||||
license_status VARCHAR(50) DEFAULT 'unlicensed',
|
||||
features TEXT DEFAULT '[]'
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS clients (
|
||||
client_id VARCHAR(100) PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
os_type VARCHAR(50) NOT NULL,
|
||||
cert_fingerprint VARCHAR(255) NOT NULL,
|
||||
status VARCHAR(50) DEFAULT 'active',
|
||||
first_seen VARCHAR(50),
|
||||
last_seen VARCHAR(50)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
username VARCHAR(100) PRIMARY KEY,
|
||||
password_hash TEXT NOT NULL,
|
||||
salt TEXT NOT NULL,
|
||||
role VARCHAR(50) NOT NULL DEFAULT 'admin',
|
||||
created_at VARCHAR(50) NOT NULL,
|
||||
totp_secret TEXT,
|
||||
totp_enabled INTEGER DEFAULT 0
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS hermes_tokens (
|
||||
token VARCHAR(255) PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
created_at VARCHAR(50) NOT NULL,
|
||||
last_used VARCHAR(50),
|
||||
status VARCHAR(50) NOT NULL DEFAULT 'active'
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS rate_limits (
|
||||
ip VARCHAR(100) NOT NULL,
|
||||
scope VARCHAR(50) NOT NULL,
|
||||
attempt_time DOUBLE PRECISION NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS server_meta (
|
||||
key VARCHAR(100) PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
""")
|
||||
# Migrations for Postgres clients table
|
||||
for col, col_def in [
|
||||
("metrics_json", "TEXT DEFAULT '{}'"),
|
||||
("cpu_percent", "DOUBLE PRECISION DEFAULT 0.0"),
|
||||
("memory_percent", "DOUBLE PRECISION DEFAULT 0.0"),
|
||||
("disk_percent_free", "DOUBLE PRECISION DEFAULT 0.0"),
|
||||
("client_version", "VARCHAR(50) DEFAULT ''"),
|
||||
("update_status", "VARCHAR(50) DEFAULT 'up_to_date'"),
|
||||
("last_update_check", "VARCHAR(50)")
|
||||
]:
|
||||
try:
|
||||
cur.execute(f"ALTER TABLE clients ADD COLUMN IF NOT EXISTS {col} {col_def}")
|
||||
except Exception:
|
||||
pass
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return True, "PostgreSQL schema successfully initialized"
|
||||
except Exception as e:
|
||||
return False, f"PostgreSQL connection/init failed: {e}"
|
||||
|
||||
@staticmethod
|
||||
def migrate_sqlite_to_postgres(
|
||||
sqlite_path: str,
|
||||
pg_config: Dict[str, Any],
|
||||
encrypt_fn: Optional[Callable[[str], str]] = None
|
||||
) -> Tuple[bool, str, Dict[str, int]]:
|
||||
"""Transfers all tables and records from SQLite to PostgreSQL with data validation and encryption."""
|
||||
if not os.path.exists(sqlite_path):
|
||||
return False, f"SQLite database not found at {sqlite_path}", {}
|
||||
|
||||
ok, init_msg = StorageEngine.init_postgres_db(pg_config)
|
||||
if not ok:
|
||||
return False, init_msg, {}
|
||||
|
||||
try:
|
||||
try:
|
||||
import psycopg2
|
||||
except ImportError:
|
||||
import psycopg as psycopg2
|
||||
|
||||
host = pg_config.get("host", "localhost")
|
||||
port = int(pg_config.get("port", 5432))
|
||||
database = pg_config.get("database", "logar")
|
||||
user = pg_config.get("user", "postgres")
|
||||
password = pg_config.get("password", "")
|
||||
sslmode = pg_config.get("sslmode", "prefer")
|
||||
timeout = int(pg_config.get("connect_timeout", 5))
|
||||
|
||||
pg_conn = psycopg2.connect(
|
||||
host=host, port=port, dbname=database, user=user, password=password, sslmode=sslmode, connect_timeout=timeout
|
||||
)
|
||||
sqlite_conn = sqlite3.connect(sqlite_path)
|
||||
sqlite_conn.row_factory = sqlite3.Row
|
||||
|
||||
migrated_counts = {}
|
||||
tables = ["active_issues", "ingest_runs", "license_config", "clients", "users", "hermes_tokens", "rate_limits", "server_meta"]
|
||||
|
||||
with pg_conn.cursor() as pg_cur:
|
||||
for tbl in tables:
|
||||
try:
|
||||
s_cur = sqlite_conn.cursor()
|
||||
s_cur.execute(f"SELECT * FROM {tbl}")
|
||||
rows = s_cur.fetchall()
|
||||
if rows:
|
||||
col_names = [col[0] for col in s_cur.description]
|
||||
cols_joined = ", ".join(col_names)
|
||||
placeholders = ", ".join(["%s"] * len(col_names))
|
||||
insert_query = f"INSERT INTO {tbl} ({cols_joined}) VALUES ({placeholders}) ON CONFLICT DO NOTHING;"
|
||||
data = []
|
||||
for r in rows:
|
||||
row_data = [r[c] for c in col_names]
|
||||
if tbl == "active_issues" and "message" in col_names and encrypt_fn:
|
||||
msg_idx = col_names.index("message")
|
||||
row_data[msg_idx] = encrypt_fn(row_data[msg_idx])
|
||||
data.append(row_data)
|
||||
|
||||
pg_cur.executemany(insert_query, data)
|
||||
migrated_counts[tbl] = len(rows)
|
||||
else:
|
||||
migrated_counts[tbl] = 0
|
||||
except Exception as tbl_err:
|
||||
print(f"[!] Warning migrating table {tbl}: {tbl_err}")
|
||||
migrated_counts[tbl] = 0
|
||||
|
||||
pg_conn.commit()
|
||||
|
||||
sqlite_conn.close()
|
||||
pg_conn.close()
|
||||
return True, "Migration completed successfully", migrated_counts
|
||||
except Exception as e:
|
||||
return False, f"Migration failed: {e}", {}
|
||||
|
||||
@staticmethod
|
||||
def get_storage_engine_info(
|
||||
db_path: Optional[str] = None,
|
||||
server_state: Optional[Dict[str, Any]] = None,
|
||||
ingest_queue: Optional[Any] = None,
|
||||
is_unlocked: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
"""Returns metadata about the active storage backend, configuration, and database telemetry."""
|
||||
state = server_state or {}
|
||||
config = state.get("config", {})
|
||||
db_type = config.get("db_type", config.get("storage_engine", "sqlite")).lower()
|
||||
target_db = db_path or config.get("db_path", StorageEngine.DEFAULT_DB_FILE)
|
||||
|
||||
table_stats = {"active_issues": 0, "ingest_runs": 0, "clients": 0, "users": 0}
|
||||
|
||||
licensed_features = []
|
||||
if target_db and os.path.exists(target_db):
|
||||
try:
|
||||
conn_l = sqlite3.connect(target_db, timeout=3.0)
|
||||
row_l = conn_l.execute("SELECT features FROM license_config WHERE id = 1").fetchone()
|
||||
conn_l.close()
|
||||
if row_l and row_l[0]:
|
||||
feat_raw = row_l[0]
|
||||
if isinstance(feat_raw, str):
|
||||
if feat_raw.startswith("["):
|
||||
licensed_features = json.loads(feat_raw)
|
||||
else:
|
||||
licensed_features = [f.strip() for f in feat_raw.split(",") if f.strip()]
|
||||
elif isinstance(feat_raw, list):
|
||||
licensed_features = list(feat_raw)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
flush_interval = ingest_queue.flush_interval if ingest_queue else 2.0
|
||||
batch_size = ingest_queue.batch_size if ingest_queue else 100
|
||||
max_queue = ingest_queue.max_queue if ingest_queue else 50000
|
||||
|
||||
if db_type in ("postgres", "postgresql"):
|
||||
pg_cfg = config.get("postgres", {})
|
||||
host = pg_cfg.get("host", "localhost")
|
||||
port = int(pg_cfg.get("port", 5432))
|
||||
dbname = pg_cfg.get("database", "logar")
|
||||
user = pg_cfg.get("user", "postgres")
|
||||
try:
|
||||
try:
|
||||
import psycopg2
|
||||
except ImportError:
|
||||
import psycopg as psycopg2
|
||||
p_conn = psycopg2.connect(
|
||||
host=host, port=port, dbname=dbname, user=user,
|
||||
password=pg_cfg.get("password", ""),
|
||||
sslmode=pg_cfg.get("sslmode", "prefer"),
|
||||
connect_timeout=int(pg_cfg.get("connect_timeout", 3))
|
||||
)
|
||||
with p_conn.cursor() as p_cur:
|
||||
for tbl in ["active_issues", "ingest_runs", "clients", "users"]:
|
||||
try:
|
||||
p_cur.execute(f"SELECT COUNT(*) FROM {tbl}")
|
||||
r = p_cur.fetchone()
|
||||
table_stats[tbl] = r[0] if r else 0
|
||||
except Exception:
|
||||
pass
|
||||
p_conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
"engine": "postgres",
|
||||
"engine_type": "postgres",
|
||||
"backend_display": "PostgreSQL Enterprise Pool",
|
||||
"connection_target": f"{user}@{host}:{port}/{dbname}",
|
||||
"status": "Enterprise Connected",
|
||||
"table_stats": table_stats,
|
||||
"is_enterprise": True,
|
||||
"feature_unlocked": is_unlocked,
|
||||
"licensed_features": licensed_features,
|
||||
"cache_mode": config.get("cache_mode", "direct"),
|
||||
"encrypt_at_rest": config.get("encrypt_at_rest", False),
|
||||
"flush_interval": flush_interval,
|
||||
"batch_size": batch_size,
|
||||
"max_queue": max_queue,
|
||||
"postgres_config": pg_cfg
|
||||
}
|
||||
else:
|
||||
size_bytes = 0
|
||||
if target_db and os.path.exists(target_db):
|
||||
try:
|
||||
size_bytes = os.path.getsize(target_db)
|
||||
conn = sqlite3.connect(target_db, timeout=3.0)
|
||||
for tbl in ["active_issues", "ingest_runs", "clients", "users"]:
|
||||
try:
|
||||
row = conn.execute(f"SELECT COUNT(*) FROM {tbl}").fetchone()
|
||||
table_stats[tbl] = row[0] if row else 0
|
||||
except Exception:
|
||||
pass
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
size_kb = round(size_bytes / 1024, 2)
|
||||
return {
|
||||
"engine": "sqlite",
|
||||
"engine_type": "sqlite",
|
||||
"backend_display": "SQLite 3 (WAL Mode)",
|
||||
"connection_target": target_db,
|
||||
"db_size_kb": size_kb,
|
||||
"status": "Local WAL Optimized",
|
||||
"table_stats": table_stats,
|
||||
"is_enterprise": False,
|
||||
"feature_unlocked": is_unlocked,
|
||||
"licensed_features": licensed_features,
|
||||
"cache_mode": config.get("cache_mode", "direct"),
|
||||
"encrypt_at_rest": config.get("encrypt_at_rest", False),
|
||||
"flush_interval": flush_interval,
|
||||
"batch_size": batch_size,
|
||||
"max_queue": max_queue,
|
||||
"postgres_config": config.get("postgres", {
|
||||
"host": "localhost",
|
||||
"port": 5432,
|
||||
"database": "logar",
|
||||
"user": "postgres",
|
||||
"password": "",
|
||||
"sslmode": "prefer",
|
||||
"connect_timeout": 5,
|
||||
"pool_size": 10
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,456 @@
|
||||
"""
|
||||
LOGAR - Central Server Update & Fleet Distribution Manager
|
||||
Handles upstream release checking, update alerting, local client package hosting,
|
||||
and client auto-update check/download endpoints.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import hashlib
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, Any, List, Optional, Tuple
|
||||
|
||||
try:
|
||||
from src.version import LOGAR_VERSION, is_newer_version, compare_versions
|
||||
except ImportError:
|
||||
from version import LOGAR_VERSION, is_newer_version, compare_versions
|
||||
|
||||
|
||||
class ServerUpdateManager:
|
||||
DEFAULT_GITEA_RELEASE_URL = "https://gitea.eibl.tech/api/v1/repos/me0nline/LOGAR/releases/latest"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
update_dir: Optional[str] = None,
|
||||
upstream_url: Optional[str] = None,
|
||||
check_enabled: bool = True
|
||||
):
|
||||
self.update_dir = update_dir or os.path.join(os.getcwd(), "updates")
|
||||
os.makedirs(self.update_dir, exist_ok=True)
|
||||
self.upstream_url = upstream_url or self.DEFAULT_GITEA_RELEASE_URL
|
||||
self.check_enabled = check_enabled
|
||||
|
||||
# Cached upstream release state
|
||||
self.last_checked_utc: Optional[str] = None
|
||||
self.latest_upstream_version: Optional[str] = None
|
||||
self.server_update_available: bool = False
|
||||
self.release_title: Optional[str] = None
|
||||
self.release_notes: Optional[str] = None
|
||||
self.release_url: Optional[str] = None
|
||||
self.release_assets: List[Dict[str, Any]] = []
|
||||
self.last_error: Optional[str] = None
|
||||
|
||||
# Hosted client packages cache
|
||||
self.hosted_client_version: str = LOGAR_VERSION
|
||||
self._sha256_cache: Dict[str, Tuple[float, int, str]] = {}
|
||||
self._package_cache: Dict[str, Dict[str, Any]] = {}
|
||||
self._package_cache_mtimes: Dict[str, float] = {}
|
||||
self._ensure_package_structure()
|
||||
|
||||
def _ensure_package_structure(self):
|
||||
"""Initializes default subdirectories for OS-specific client packages."""
|
||||
for os_name in ["windows", "linux"]:
|
||||
os.makedirs(os.path.join(self.update_dir, os_name), exist_ok=True)
|
||||
|
||||
@classmethod
|
||||
def calculate_sha256(cls, filepath: str) -> str:
|
||||
"""Computes the SHA-256 digest of a local file with stat-based memoization."""
|
||||
if not os.path.exists(filepath):
|
||||
return ""
|
||||
try:
|
||||
stat = os.stat(filepath)
|
||||
mtime = stat.st_mtime
|
||||
size = stat.st_size
|
||||
if hasattr(cls, "_global_sha256_cache"):
|
||||
cached = cls._global_sha256_cache.get(filepath)
|
||||
if cached and cached[0] == mtime and cached[1] == size:
|
||||
return cached[2]
|
||||
except Exception:
|
||||
mtime = 0.0
|
||||
size = 0
|
||||
|
||||
h = hashlib.sha256()
|
||||
with open(filepath, "rb") as f:
|
||||
while chunk := f.read(65536):
|
||||
h.update(chunk)
|
||||
digest = h.hexdigest().lower()
|
||||
|
||||
if not hasattr(cls, "_global_sha256_cache"):
|
||||
cls._global_sha256_cache = {}
|
||||
cls._global_sha256_cache[filepath] = (mtime, size, digest)
|
||||
return digest
|
||||
|
||||
def set_hosted_client_version(self, version_str: str):
|
||||
"""Sets the version tag for currently hosted client packages."""
|
||||
if version_str:
|
||||
self.hosted_client_version = version_str.strip().lstrip("v")
|
||||
self._package_cache.clear()
|
||||
|
||||
def invalidate_package_cache(self, os_type: Optional[str] = None):
|
||||
"""Clears in-memory package cache."""
|
||||
if os_type:
|
||||
self._package_cache.pop(os_type.lower(), None)
|
||||
self._package_cache_mtimes.pop(os_type.lower(), None)
|
||||
else:
|
||||
self._package_cache.clear()
|
||||
self._package_cache_mtimes.clear()
|
||||
|
||||
def register_client_package(
|
||||
self,
|
||||
os_type: str,
|
||||
file_path: str,
|
||||
version: Optional[str] = None,
|
||||
filename: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Saves/copies a client binary package into the local update repository.
|
||||
Computes SHA-256 checksum and updates metadata.
|
||||
"""
|
||||
norm_os = os_type.lower()
|
||||
if norm_os not in ["windows", "linux"]:
|
||||
raise ValueError(f"Unsupported OS type: {os_type}")
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"Package file not found: {file_path}")
|
||||
|
||||
target_dir = os.path.join(self.update_dir, norm_os)
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
|
||||
raw_name = filename or os.path.basename(file_path)
|
||||
target_name = os.path.basename(raw_name)
|
||||
dest_path = os.path.join(target_dir, target_name)
|
||||
|
||||
if os.path.abspath(file_path) != os.path.abspath(dest_path):
|
||||
with open(file_path, "rb") as src_f, open(dest_path, "wb") as dst_f:
|
||||
while chunk := src_f.read(65536):
|
||||
dst_f.write(chunk)
|
||||
|
||||
sha256 = self.calculate_sha256(dest_path)
|
||||
pkg_version = version or self.hosted_client_version or LOGAR_VERSION
|
||||
self.set_hosted_client_version(pkg_version)
|
||||
|
||||
# Write manifest file next to package
|
||||
manifest = {
|
||||
"os": norm_os,
|
||||
"filename": target_name,
|
||||
"version": pkg_version,
|
||||
"sha256": sha256,
|
||||
"size_bytes": os.path.getsize(dest_path),
|
||||
"updated_at": datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
}
|
||||
manifest_path = os.path.join(target_dir, "manifest.json")
|
||||
with open(manifest_path, "w", encoding="utf-8") as mf:
|
||||
json.dump(manifest, mf, indent=2)
|
||||
|
||||
# Update cache directly
|
||||
cached_entry = dict(manifest)
|
||||
cached_entry["file_path"] = dest_path
|
||||
cached_entry["exists"] = True
|
||||
self._package_cache[norm_os] = cached_entry
|
||||
self._package_cache_mtimes[norm_os] = os.path.getmtime(dest_path)
|
||||
|
||||
return manifest
|
||||
|
||||
def get_hosted_packages(self, force_refresh: bool = False) -> Dict[str, Dict[str, Any]]:
|
||||
"""
|
||||
Returns info on currently hosted client packages for Windows and Linux.
|
||||
Uses in-memory cache validated against file modification times to minimize disk I/O.
|
||||
"""
|
||||
result: Dict[str, Dict[str, Any]] = {}
|
||||
for os_name in ["windows", "linux"]:
|
||||
manifest_path = os.path.join(self.update_dir, os_name, "manifest.json")
|
||||
target_file = None
|
||||
|
||||
if not force_refresh and os_name in self._package_cache:
|
||||
cached = self._package_cache[os_name]
|
||||
tf = cached.get("file_path")
|
||||
if tf and os.path.exists(tf):
|
||||
try:
|
||||
cur_mtime = os.path.getmtime(tf)
|
||||
if cur_mtime == self._package_cache_mtimes.get(os_name):
|
||||
result[os_name] = cached
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if os.path.exists(manifest_path):
|
||||
try:
|
||||
with open(manifest_path, "r", encoding="utf-8") as mf:
|
||||
info = json.load(mf)
|
||||
target_file = os.path.join(self.update_dir, os_name, info.get("filename", ""))
|
||||
if os.path.exists(target_file):
|
||||
info["exists"] = True
|
||||
info["file_path"] = target_file
|
||||
result[os_name] = info
|
||||
self._package_cache[os_name] = info
|
||||
self._package_cache_mtimes[os_name] = os.path.getmtime(target_file)
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback scan in update_dir/{os_name}
|
||||
os_dir = os.path.join(self.update_dir, os_name)
|
||||
found_pkg = None
|
||||
if os.path.exists(os_dir):
|
||||
for f in os.listdir(os_dir):
|
||||
if f.endswith((".exe", ".bin", ".zip", ".tar.gz", ".sh")) and not f.startswith("."):
|
||||
found_pkg = os.path.join(os_dir, f)
|
||||
break
|
||||
|
||||
# Secondary fallback: search project dist/
|
||||
if not found_pkg:
|
||||
dist_dir = os.path.join(os.path.dirname(self.update_dir), "dist")
|
||||
if os.path.exists(dist_dir):
|
||||
patterns = [
|
||||
"LOGAR_Client_Setup.exe", "LOGAR_Client.exe", "Win_Client.exe"
|
||||
] if os_name == "windows" else [
|
||||
"Linux_Client.bin", "LOGAR_Client_Linux", "install_linux_client.sh"
|
||||
]
|
||||
for pat in patterns:
|
||||
candidate = os.path.join(dist_dir, pat)
|
||||
if os.path.exists(candidate):
|
||||
found_pkg = candidate
|
||||
break
|
||||
|
||||
if found_pkg and os.path.exists(found_pkg):
|
||||
sha256 = self.calculate_sha256(found_pkg)
|
||||
pkg_data = {
|
||||
"os": os_name,
|
||||
"filename": os.path.basename(found_pkg),
|
||||
"version": self.hosted_client_version or LOGAR_VERSION,
|
||||
"sha256": sha256,
|
||||
"size_bytes": os.path.getsize(found_pkg),
|
||||
"file_path": found_pkg,
|
||||
"exists": True,
|
||||
"updated_at": datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
}
|
||||
result[os_name] = pkg_data
|
||||
self._package_cache[os_name] = pkg_data
|
||||
self._package_cache_mtimes[os_name] = os.path.getmtime(found_pkg)
|
||||
else:
|
||||
pkg_data = {
|
||||
"os": os_name,
|
||||
"filename": None,
|
||||
"version": self.hosted_client_version or LOGAR_VERSION,
|
||||
"sha256": "",
|
||||
"size_bytes": 0,
|
||||
"file_path": None,
|
||||
"exists": False
|
||||
}
|
||||
result[os_name] = pkg_data
|
||||
self._package_cache[os_name] = pkg_data
|
||||
self._package_cache_mtimes[os_name] = 0.0
|
||||
|
||||
return result
|
||||
|
||||
def get_client_package_path(self, os_type: str) -> Optional[Tuple[str, str, str]]:
|
||||
"""
|
||||
Returns (file_path, filename, sha256) for downloading a client update package.
|
||||
"""
|
||||
packages = self.get_hosted_packages()
|
||||
pkg_info = packages.get(os_type.lower())
|
||||
if pkg_info and pkg_info.get("exists") and pkg_info.get("file_path"):
|
||||
return pkg_info["file_path"], pkg_info["filename"], pkg_info["sha256"]
|
||||
return None
|
||||
|
||||
def check_client_update(
|
||||
self,
|
||||
client_id: str,
|
||||
os_type: str,
|
||||
current_version: Optional[str],
|
||||
arch: str = "x86_64"
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Evaluates whether a client requires an update based on its reported version
|
||||
and the server's hosted client package version.
|
||||
"""
|
||||
norm_os = (os_type or "windows").lower()
|
||||
packages = self.get_hosted_packages()
|
||||
pkg = packages.get(norm_os, {})
|
||||
|
||||
target_version = pkg.get("version") or self.hosted_client_version or LOGAR_VERSION
|
||||
has_pkg = bool(pkg.get("exists"))
|
||||
client_ver = (current_version or "0.0.0.0").strip().lstrip("v")
|
||||
|
||||
# An update is available if the hosted version is strictly newer than the client version
|
||||
update_available = has_pkg and is_newer_version(target_version, client_ver)
|
||||
|
||||
return {
|
||||
"update_available": update_available,
|
||||
"client_id": client_id,
|
||||
"os": norm_os,
|
||||
"current_version": client_ver,
|
||||
"latest_version": target_version,
|
||||
"sha256": pkg.get("sha256", "") if update_available else "",
|
||||
"download_url": f"/api/client/update/download?os={norm_os}" if update_available else "",
|
||||
"size_bytes": pkg.get("size_bytes", 0) if update_available else 0,
|
||||
"filename": pkg.get("filename", "") if update_available else "",
|
||||
"release_notes": self.release_notes or f"LOGAR {target_version} update for {norm_os.capitalize()}.",
|
||||
"mandatory": False
|
||||
}
|
||||
|
||||
def download_and_cache_client_packages(
|
||||
self,
|
||||
assets: Optional[List[Dict[str, Any]]] = None,
|
||||
target_version: Optional[str] = None,
|
||||
timeout: int = 60
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Downloads OS client binary packages from upstream release assets and caches them locally.
|
||||
Ensures the Hub acts as a caching distribution point so edge clients don't contact external servers.
|
||||
"""
|
||||
release_assets = assets if assets is not None else self.release_assets
|
||||
version = target_version or self.latest_upstream_version or self.hosted_client_version or LOGAR_VERSION
|
||||
results = {"windows": False, "linux": False, "downloaded": []}
|
||||
|
||||
if not release_assets:
|
||||
return results
|
||||
|
||||
temp_dir = os.path.join(self.update_dir, "temp")
|
||||
os.makedirs(temp_dir, exist_ok=True)
|
||||
|
||||
for asset in release_assets:
|
||||
name = (asset.get("name") or "").lower()
|
||||
download_url = asset.get("download_url")
|
||||
if not download_url:
|
||||
continue
|
||||
|
||||
target_os = None
|
||||
if ("win" in name or name.endswith(".exe") or "setup" in name) and not results["windows"]:
|
||||
target_os = "windows"
|
||||
elif ("linux" in name or name.endswith((".bin", ".sh")) or "client_linux" in name) and not results["linux"]:
|
||||
target_os = "linux"
|
||||
|
||||
if not target_os:
|
||||
continue
|
||||
|
||||
# Check if matching version is already cached locally
|
||||
existing_pkg = self.get_hosted_packages().get(target_os)
|
||||
if existing_pkg and existing_pkg.get("exists") and existing_pkg.get("version") == version:
|
||||
results[target_os] = True
|
||||
continue
|
||||
|
||||
filename = os.path.basename(asset.get("name") or f"client_{target_os}")
|
||||
temp_file = os.path.join(temp_dir, filename)
|
||||
|
||||
try:
|
||||
print(f"[*] Fleet Hub caching upstream client package for {target_os}: {filename} ({download_url})...")
|
||||
req = urllib.request.Request(
|
||||
download_url,
|
||||
headers={"User-Agent": f"LOGAR-Server-Hub/{LOGAR_VERSION}"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
if resp.status == 200:
|
||||
with open(temp_file, "wb") as f:
|
||||
while chunk := resp.read(65536):
|
||||
f.write(chunk)
|
||||
|
||||
# Register package in Hub repository
|
||||
manifest = self.register_client_package(
|
||||
os_type=target_os,
|
||||
file_path=temp_file,
|
||||
version=version,
|
||||
filename=filename
|
||||
)
|
||||
results[target_os] = True
|
||||
results["downloaded"].append({
|
||||
"os": target_os,
|
||||
"filename": filename,
|
||||
"version": version,
|
||||
"sha256": manifest.get("sha256")
|
||||
})
|
||||
print(f"[+] Successfully cached {target_os} client package v{version} (SHA-256: {manifest.get('sha256')[:12]}...).")
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Failed to download and cache upstream client asset '{filename}': {e}")
|
||||
finally:
|
||||
if os.path.exists(temp_file):
|
||||
try:
|
||||
os.remove(temp_file)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if results["downloaded"]:
|
||||
self.hosted_client_version = version
|
||||
|
||||
return results
|
||||
|
||||
def check_upstream_updates(self, force: bool = False, auto_download_clients: bool = True, timeout: int = 15) -> Dict[str, Any]:
|
||||
"""
|
||||
Queries the upstream repository release API (Gitea / GitHub / JSON manifest)
|
||||
to check if a newer server or client release is available, and automatically caches client packages.
|
||||
"""
|
||||
if not self.check_enabled and not force:
|
||||
return self.get_status()
|
||||
|
||||
now_utc = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
self.last_checked_utc = now_utc
|
||||
self.last_error = None
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
self.upstream_url,
|
||||
headers={
|
||||
"User-Agent": f"LOGAR-Server/{LOGAR_VERSION}",
|
||||
"Accept": "application/json"
|
||||
}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
if resp.status == 200:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
raw_tag = data.get("tag_name") or data.get("name") or data.get("version") or ""
|
||||
upstream_ver = raw_tag.strip().lstrip("v")
|
||||
|
||||
self.latest_upstream_version = upstream_ver
|
||||
self.release_title = data.get("name") or raw_tag
|
||||
self.release_notes = data.get("body") or data.get("notes") or ""
|
||||
self.release_url = data.get("html_url") or data.get("url") or ""
|
||||
|
||||
# Process assets list
|
||||
assets = []
|
||||
for a in data.get("assets", []):
|
||||
assets.append({
|
||||
"name": a.get("name"),
|
||||
"size": a.get("size"),
|
||||
"download_url": a.get("browser_download_url") or a.get("download_url")
|
||||
})
|
||||
self.release_assets = assets
|
||||
|
||||
# Compare version against current LOGAR_VERSION
|
||||
self.server_update_available = is_newer_version(upstream_ver, LOGAR_VERSION)
|
||||
if self.server_update_available:
|
||||
print(f"[!] 🔔 LOGAR Server Update Available: Current v{LOGAR_VERSION} -> Upstream v{upstream_ver}")
|
||||
|
||||
# Automatically download and cache client update packages on the Hub
|
||||
if auto_download_clients and assets:
|
||||
# Check if hosted packages need updating
|
||||
hosted = self.get_hosted_packages()
|
||||
needs_win = not hosted.get("windows", {}).get("exists") or is_newer_version(upstream_ver, hosted.get("windows", {}).get("version", "0.0.0"))
|
||||
needs_linux = not hosted.get("linux", {}).get("exists") or is_newer_version(upstream_ver, hosted.get("linux", {}).get("version", "0.0.0"))
|
||||
if needs_win or needs_linux or force:
|
||||
self.download_and_cache_client_packages(assets=assets, target_version=upstream_ver)
|
||||
else:
|
||||
self.last_error = f"Upstream returned HTTP status {resp.status}"
|
||||
except Exception as e:
|
||||
self.last_error = f"Failed to check upstream updates: {e}"
|
||||
|
||||
return self.get_status()
|
||||
|
||||
def get_status(self) -> Dict[str, Any]:
|
||||
"""Returns structured update status for WebUI and REST APIs."""
|
||||
return {
|
||||
"current_server_version": LOGAR_VERSION,
|
||||
"latest_upstream_version": self.latest_upstream_version or LOGAR_VERSION,
|
||||
"server_update_available": self.server_update_available,
|
||||
"hosted_client_version": self.hosted_client_version or LOGAR_VERSION,
|
||||
"release_title": self.release_title or "",
|
||||
"release_notes": self.release_notes or "",
|
||||
"release_url": self.release_url or "",
|
||||
"release_assets": self.release_assets,
|
||||
"last_checked_utc": self.last_checked_utc,
|
||||
"last_error": self.last_error,
|
||||
"check_enabled": self.check_enabled,
|
||||
"hosted_packages": self.get_hosted_packages()
|
||||
}
|
||||
+1954
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,11 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
_CUR = os.path.dirname(__file__)
|
||||
_ROOT = os.path.abspath(os.path.join(_CUR, ".."))
|
||||
|
||||
for _p in [_CUR, _ROOT]:
|
||||
if _p and _p not in sys.path:
|
||||
sys.path.insert(0, _p)
|
||||
|
||||
from src.hub.ingest_queue import *
|
||||
@@ -0,0 +1,11 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
_CUR = os.path.dirname(__file__)
|
||||
_ROOT = os.path.abspath(os.path.join(_CUR, ".."))
|
||||
|
||||
for _p in [_CUR, _ROOT]:
|
||||
if _p and _p not in sys.path:
|
||||
sys.path.insert(0, _p)
|
||||
|
||||
from src.hub.license_manager import *
|
||||
+6
-537
@@ -1,542 +1,11 @@
|
||||
"""
|
||||
LOGAR - Unified Cross-Platform Log Collector & State Management
|
||||
Provides modular log collection classes and state management for Windows and Linux,
|
||||
reusable across Client Agents (Win_Client, Linux_Client) and Server Hubs (Windows & Linux).
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import socket
|
||||
import subprocess
|
||||
import warnings
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Optional, List, Dict, Any, Set
|
||||
|
||||
# Suppress cryptography / pgpy deprecation notices
|
||||
warnings.filterwarnings("ignore")
|
||||
_CUR = os.path.dirname(__file__)
|
||||
_ROOT = os.path.abspath(os.path.join(_CUR, ".."))
|
||||
|
||||
try:
|
||||
import win32evtlog
|
||||
except ImportError:
|
||||
win32evtlog = None
|
||||
for _p in [_CUR, _ROOT]:
|
||||
if _p and _p not in sys.path:
|
||||
sys.path.insert(0, _p)
|
||||
|
||||
CONFIG_FILE_NAME = "client_config.json"
|
||||
STATE_FILE_NAME = "client_state.json"
|
||||
|
||||
|
||||
def get_machine_identifier() -> str:
|
||||
"""
|
||||
Returns the fully qualified hostname of the machine sending/collecting logs.
|
||||
Cross-platform resolution:
|
||||
1. FQDN (if non-local)
|
||||
2. Windows USERDNSDOMAIN (on Windows)
|
||||
3. Linux /etc/resolv.conf domain/search directive (on Linux)
|
||||
4. Reverse DNS lookup
|
||||
5. socket.gethostname() fallback
|
||||
"""
|
||||
fqdn = socket.getfqdn()
|
||||
if fqdn and "." in fqdn and not fqdn.startswith("localhost"):
|
||||
return fqdn
|
||||
|
||||
hostname = socket.gethostname()
|
||||
|
||||
# Windows DNS domain
|
||||
if sys.platform == "win32":
|
||||
user_dns_domain = os.environ.get("USERDNSDOMAIN")
|
||||
if user_dns_domain and user_dns_domain.lower() != hostname.lower():
|
||||
return f"{hostname}.{user_dns_domain.lower()}"
|
||||
|
||||
# Linux resolv.conf search / domain
|
||||
if sys.platform.startswith("linux") or os.path.exists("/etc/resolv.conf"):
|
||||
try:
|
||||
with open("/etc/resolv.conf", "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
parts = line.strip().split()
|
||||
if parts and parts[0] in ["domain", "search"] and len(parts) > 1:
|
||||
domain = parts[1]
|
||||
if domain and not domain.startswith("."):
|
||||
return f"{hostname}.{domain}"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Reverse DNS lookup
|
||||
try:
|
||||
host_ip = socket.gethostbyname(hostname)
|
||||
canonical_name = socket.gethostbyaddr(host_ip)[0]
|
||||
if canonical_name and "." in canonical_name and not canonical_name.startswith("localhost"):
|
||||
return canonical_name
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return hostname
|
||||
|
||||
|
||||
def is_path_writable(filepath: str) -> bool:
|
||||
"""Checks if a file or directory path is actually writable by the current process."""
|
||||
try:
|
||||
if os.path.exists(filepath):
|
||||
if os.path.isdir(filepath):
|
||||
test_f = os.path.join(filepath, f".perm_{os.getpid()}")
|
||||
with open(test_f, "w") as f:
|
||||
f.write("1")
|
||||
os.remove(test_f)
|
||||
return True
|
||||
else:
|
||||
with open(filepath, "a") as f:
|
||||
pass
|
||||
return True
|
||||
else:
|
||||
parent = os.path.dirname(os.path.abspath(filepath)) or "."
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
test_f = os.path.join(parent, f".perm_{os.getpid()}")
|
||||
with open(test_f, "w") as f:
|
||||
f.write("1")
|
||||
os.remove(test_f)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def get_default_data_dir(app_name: str = "LOGAR") -> str:
|
||||
"""Returns a suitable writable application data directory."""
|
||||
if sys.platform == "win32":
|
||||
# 1. ProgramData (C:\ProgramData\LOGAR)
|
||||
prog_data = os.environ.get("ProgramData", r"C:\ProgramData")
|
||||
target_prog = os.path.join(prog_data, app_name)
|
||||
if is_path_writable(target_prog):
|
||||
return target_prog
|
||||
|
||||
# 2. LocalAppData (C:\Users\<user>\AppData\Local\LOGAR)
|
||||
local_app = os.environ.get("LOCALAPPDATA") or os.path.expanduser(r"~\AppData\Local")
|
||||
target_local = os.path.join(local_app, app_name)
|
||||
if is_path_writable(target_local):
|
||||
return target_local
|
||||
|
||||
# 3. Temp directory (%TEMP%\LOGAR)
|
||||
temp_dir = os.path.join(os.environ.get("TEMP", r"C:\Windows\Temp"), app_name)
|
||||
if is_path_writable(temp_dir):
|
||||
return temp_dir
|
||||
else:
|
||||
# Linux / Unix candidates: /var/lib/<app_name>, /tmp/<app_name>, ~/.local/share/<app_name>
|
||||
var_lib = f"/var/lib/{app_name.lower()}"
|
||||
if is_path_writable(var_lib):
|
||||
return var_lib
|
||||
user_share = os.path.expanduser(f"~/.local/share/{app_name.lower()}")
|
||||
if is_path_writable(user_share):
|
||||
return user_share
|
||||
|
||||
cwd = os.getcwd()
|
||||
if is_path_writable(cwd):
|
||||
return cwd
|
||||
return os.path.expanduser("~")
|
||||
|
||||
|
||||
def resolve_writable_path(path: str, app_name: str = "LOGAR", default_filename: str = "client_state.json") -> str:
|
||||
"""Resolves a file or directory path, ensuring it is writable or falling back to a writable data directory."""
|
||||
if not path:
|
||||
path = default_filename
|
||||
|
||||
if os.path.isabs(path) and is_path_writable(path):
|
||||
return path
|
||||
|
||||
if not os.path.isabs(path) and "Program Files" not in os.getcwd() and is_path_writable(os.getcwd()):
|
||||
return os.path.abspath(path)
|
||||
|
||||
data_dir = get_default_data_dir(app_name)
|
||||
fname = os.path.basename(path)
|
||||
return os.path.join(data_dir, fname)
|
||||
|
||||
|
||||
def get_state_path(config_path: str, custom_state_path: Optional[str] = None) -> str:
|
||||
if custom_state_path:
|
||||
return custom_state_path
|
||||
config_dir = os.path.dirname(os.path.abspath(config_path))
|
||||
if sys.platform == "win32" and ("Program Files" in config_dir or not is_path_writable(config_dir)):
|
||||
return os.path.join(get_default_data_dir("LOGAR"), STATE_FILE_NAME)
|
||||
return os.path.join(config_dir, STATE_FILE_NAME)
|
||||
|
||||
|
||||
def load_state(state_path: str) -> dict:
|
||||
"""Loads state tracking dictionary from disk."""
|
||||
if os.path.exists(state_path):
|
||||
try:
|
||||
with open(state_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Failed to read state file '{state_path}': {e}")
|
||||
return {}
|
||||
return {}
|
||||
|
||||
|
||||
def save_state(state_path: str, state: dict):
|
||||
"""Atomically saves state dictionary to disk."""
|
||||
try:
|
||||
os.makedirs(os.path.dirname(os.path.abspath(state_path)), exist_ok=True)
|
||||
temp_path = f"{state_path}.tmp"
|
||||
with open(temp_path, "w", encoding="utf-8") as f:
|
||||
json.dump(state, f, indent=2)
|
||||
os.replace(temp_path, state_path)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Could not save client state to '{state_path}': {e}")
|
||||
|
||||
|
||||
def commit_state(state: dict, state_path: str):
|
||||
"""
|
||||
Commits staged cursor/record state keys to persistent fields and writes to state_path.
|
||||
Supports both Windows record numbers and Linux cursors/timestamps.
|
||||
"""
|
||||
if "new_last_record_number" in state:
|
||||
val = state.pop("new_last_record_number")
|
||||
if val:
|
||||
state["last_record_number"] = val
|
||||
if "new_sent_record_ids" in state:
|
||||
state["sent_record_ids"] = state.pop("new_sent_record_ids")
|
||||
if "new_last_cursor" in state:
|
||||
val = state.pop("new_last_cursor")
|
||||
if val:
|
||||
state["last_cursor"] = val
|
||||
if "new_last_timestamp_us" in state:
|
||||
val = state.pop("new_last_timestamp_us")
|
||||
if val:
|
||||
state["last_timestamp_us"] = val
|
||||
if "new_sent_cursors" in state:
|
||||
state["sent_cursors"] = state.pop("new_sent_cursors")
|
||||
save_state(state_path, state)
|
||||
|
||||
|
||||
class BaseLogCollector(ABC):
|
||||
"""
|
||||
Abstract base class for platform-specific edge and hub log collectors.
|
||||
Manages state persistence, machine identifiers, lookback windows, and normalization.
|
||||
"""
|
||||
|
||||
def __init__(self, state: Optional[dict] = None, machine_id: Optional[str] = None):
|
||||
self.state = state if state is not None else {}
|
||||
self.machine_id = machine_id or get_machine_identifier()
|
||||
|
||||
@abstractmethod
|
||||
def collect_logs(self, hours: int = 24) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Collects normalized logs within the given lookback window.
|
||||
Returns a list of dicts:
|
||||
[
|
||||
{
|
||||
"server": str,
|
||||
"os_type": "windows"|"linux",
|
||||
"signature": str,
|
||||
"severity": "ERROR"|"WARNING"|"INFO",
|
||||
"message": str
|
||||
},
|
||||
...
|
||||
]
|
||||
"""
|
||||
pass
|
||||
|
||||
def commit(self, state_path: Optional[str] = None):
|
||||
"""Commits staged state keys and optionally writes to disk."""
|
||||
if state_path:
|
||||
commit_state(self.state, state_path)
|
||||
else:
|
||||
if "new_last_record_number" in self.state:
|
||||
val = self.state.pop("new_last_record_number")
|
||||
if val:
|
||||
self.state["last_record_number"] = val
|
||||
if "new_sent_record_ids" in self.state:
|
||||
self.state["sent_record_ids"] = self.state.pop("new_sent_record_ids")
|
||||
if "new_last_cursor" in self.state:
|
||||
val = self.state.pop("new_last_cursor")
|
||||
if val:
|
||||
self.state["last_cursor"] = val
|
||||
if "new_last_timestamp_us" in self.state:
|
||||
val = self.state.pop("new_last_timestamp_us")
|
||||
if val:
|
||||
self.state["last_timestamp_us"] = val
|
||||
if "new_sent_cursors" in self.state:
|
||||
self.state["sent_cursors"] = self.state.pop("new_sent_cursors")
|
||||
|
||||
|
||||
class WindowsLogCollector(BaseLogCollector):
|
||||
"""
|
||||
Windows Event Log Collector.
|
||||
Reads Application and System (or custom channels) backwards.
|
||||
Edge Filtering: Retains INFO, WARNING, and ERROR. Drops Audit and Debug noise.
|
||||
State Tracking: Skips events older than lookback window and already sent records.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
state: Optional[dict] = None,
|
||||
machine_id: Optional[str] = None,
|
||||
channels: Optional[List[str]] = None
|
||||
):
|
||||
super().__init__(state=state, machine_id=machine_id)
|
||||
self.channels = channels or ["Application"]
|
||||
|
||||
def collect_logs(self, hours: int = 24) -> List[Dict[str, Any]]:
|
||||
if win32evtlog is None:
|
||||
return []
|
||||
|
||||
logs = []
|
||||
cutoff_time = datetime.now() - timedelta(hours=hours)
|
||||
|
||||
last_record_number = 0
|
||||
sent_record_ids: Set[str] = set()
|
||||
if self.state:
|
||||
last_record_number = int(self.state.get("last_record_number", 0))
|
||||
sent_record_ids = set(self.state.get("sent_record_ids", []))
|
||||
|
||||
# Windows Event Log EventTypes:
|
||||
# 1: EVENTLOG_ERROR_TYPE -> ERROR
|
||||
# 2: EVENTLOG_WARNING_TYPE -> WARNING
|
||||
# 4: EVENTLOG_INFORMATION_TYPE -> INFO
|
||||
sev_map = {
|
||||
1: "ERROR",
|
||||
2: "WARNING",
|
||||
4: "INFO"
|
||||
}
|
||||
|
||||
newest_record_number = 0
|
||||
collected_record_ids = []
|
||||
|
||||
flags = win32evtlog.EVENTLOG_BACKWARDS_READ | win32evtlog.EVENTLOG_SEQUENTIAL_READ
|
||||
|
||||
for log_type in self.channels:
|
||||
try:
|
||||
hand = win32evtlog.OpenEventLog("localhost", log_type)
|
||||
except Exception as e:
|
||||
continue
|
||||
|
||||
while True:
|
||||
try:
|
||||
events = win32evtlog.ReadEventLog(hand, flags, 0)
|
||||
except Exception:
|
||||
break
|
||||
if not events:
|
||||
break
|
||||
|
||||
for event in events:
|
||||
rec_num = int(event.RecordNumber)
|
||||
if newest_record_number == 0:
|
||||
newest_record_number = rec_num
|
||||
|
||||
# 1. Skip entries older than lookback window
|
||||
if event.TimeGenerated < cutoff_time:
|
||||
break
|
||||
|
||||
# 2. Skip already sent events
|
||||
if last_record_number > 0 and newest_record_number >= last_record_number:
|
||||
if rec_num <= last_record_number:
|
||||
break
|
||||
|
||||
rec_id = f"{log_type}:{rec_num}:{event.TimeGenerated.isoformat()}" if len(self.channels) > 1 else f"{rec_num}:{event.TimeGenerated.isoformat()}"
|
||||
if rec_id in sent_record_ids:
|
||||
continue
|
||||
|
||||
# Filter: upload everything from INFO to ERROR
|
||||
if event.EventType in sev_map:
|
||||
msg = " ".join(event.StringInserts) if event.StringInserts else "Event Log Entry"
|
||||
sig = f"{log_type}/{event.SourceName or 'Windows-Event'}" if len(self.channels) > 1 else (event.SourceName or "Windows-Event")
|
||||
logs.append({
|
||||
"server": self.machine_id,
|
||||
"os_type": "windows",
|
||||
"signature": sig,
|
||||
"severity": sev_map[event.EventType],
|
||||
"message": msg[:2048]
|
||||
})
|
||||
collected_record_ids.append(rec_id)
|
||||
|
||||
if events[-1].TimeGenerated < cutoff_time:
|
||||
break
|
||||
if last_record_number > 0 and newest_record_number >= last_record_number and events[-1].RecordNumber <= last_record_number:
|
||||
break
|
||||
|
||||
try:
|
||||
win32evtlog.CloseEventLog(hand)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if self.state is not None:
|
||||
target_rec = max(newest_record_number, last_record_number)
|
||||
self.state["new_last_record_number"] = target_rec
|
||||
self.state["new_sent_record_ids"] = (list(sent_record_ids) + collected_record_ids)[-1000:]
|
||||
# Also update direct keys for immediate server-side ingestion
|
||||
self.state["last_record_number"] = target_rec
|
||||
self.state["sent_record_ids"] = self.state["new_sent_record_ids"]
|
||||
self.state["last_run_timestamp"] = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
return logs
|
||||
|
||||
|
||||
class LinuxLogCollector(BaseLogCollector):
|
||||
"""
|
||||
Linux Systemd Journal / Syslog Log Collector.
|
||||
Collects info, warnings, and errors from journalctl (or syslog fallback).
|
||||
Edge Filtering: Retains INFO, WARNING, and ERROR. Drops DEBUG noise.
|
||||
State Tracking: Skips events older than lookback window and already sent cursors.
|
||||
"""
|
||||
|
||||
def __init__(self, state: Optional[dict] = None, machine_id: Optional[str] = None):
|
||||
super().__init__(state=state, machine_id=machine_id)
|
||||
|
||||
def collect_logs(self, hours: int = 24) -> List[Dict[str, Any]]:
|
||||
last_cursor = None
|
||||
last_timestamp_us = 0.0
|
||||
sent_cursors: Set[str] = set()
|
||||
if self.state:
|
||||
last_cursor = self.state.get("last_cursor")
|
||||
try:
|
||||
last_timestamp_us = float(self.state.get("last_timestamp_us", 0))
|
||||
except (ValueError, TypeError):
|
||||
last_timestamp_us = 0.0
|
||||
sent_cursors = set(self.state.get("sent_cursors", []))
|
||||
|
||||
cmd = ["journalctl", "--since", f"{hours} hours ago", "-p", "info", "--output=json"]
|
||||
result = None
|
||||
|
||||
if last_cursor:
|
||||
cmd_with_cursor = ["journalctl", "--since", f"{hours} hours ago", "--after-cursor", str(last_cursor), "-p", "info", "--output=json"]
|
||||
try:
|
||||
res = subprocess.run(cmd_with_cursor, capture_output=True, text=True, check=False)
|
||||
if res.returncode == 0:
|
||||
result = res
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if result is None:
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||
except FileNotFoundError:
|
||||
# journalctl not present, attempt fallback to /var/log/syslog or messages
|
||||
return self._collect_syslog_fallback(hours=hours)
|
||||
except Exception as e:
|
||||
print(f"[!] Error running journalctl: {e}")
|
||||
return []
|
||||
|
||||
if not result or result.returncode != 0:
|
||||
return []
|
||||
|
||||
logs = []
|
||||
cutoff_epoch_us = (datetime.now(timezone.utc) - timedelta(hours=hours)).timestamp() * 1_000_000
|
||||
|
||||
newest_cursor = None
|
||||
newest_timestamp_us = last_timestamp_us
|
||||
collected_cursors = []
|
||||
|
||||
for line in result.stdout.splitlines():
|
||||
line_str = line.strip()
|
||||
if not line_str:
|
||||
continue
|
||||
try:
|
||||
entry = json.loads(line_str)
|
||||
entry_cursor = entry.get("__CURSOR")
|
||||
entry_ts_us_raw = entry.get("__REALTIME_TIMESTAMP")
|
||||
|
||||
entry_ts_us = 0.0
|
||||
if entry_ts_us_raw:
|
||||
try:
|
||||
entry_ts_us = float(entry_ts_us_raw)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# 1. Skip entries older than lookback window
|
||||
if entry_ts_us and entry_ts_us < cutoff_epoch_us:
|
||||
continue
|
||||
|
||||
# 2. Skip already sent events
|
||||
if entry_cursor and (entry_cursor in sent_cursors or entry_cursor == last_cursor):
|
||||
continue
|
||||
if last_timestamp_us > 0 and entry_ts_us > 0 and entry_ts_us < last_timestamp_us:
|
||||
continue
|
||||
|
||||
# Advance tracking
|
||||
if entry_cursor:
|
||||
newest_cursor = entry_cursor
|
||||
collected_cursors.append(entry_cursor)
|
||||
if entry_ts_us > newest_timestamp_us:
|
||||
newest_timestamp_us = entry_ts_us
|
||||
|
||||
priority = int(entry.get("PRIORITY", "6"))
|
||||
if priority > 6:
|
||||
continue
|
||||
|
||||
if priority <= 3:
|
||||
sev = "ERROR"
|
||||
elif priority in (4, 5):
|
||||
sev = "WARNING"
|
||||
else:
|
||||
sev = "INFO"
|
||||
|
||||
logs.append({
|
||||
"server": self.machine_id,
|
||||
"os_type": "linux",
|
||||
"signature": entry.get("SYSLOG_IDENTIFIER", "unknown"),
|
||||
"severity": sev,
|
||||
"message": entry.get("MESSAGE", "")[:2048]
|
||||
})
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
|
||||
if self.state is not None:
|
||||
self.state["new_last_cursor"] = newest_cursor or last_cursor
|
||||
self.state["new_last_timestamp_us"] = max(newest_timestamp_us, last_timestamp_us)
|
||||
self.state["new_sent_cursors"] = (list(sent_cursors) + collected_cursors)[-1000:]
|
||||
# Also update direct keys for immediate server-side ingestion
|
||||
self.state["last_cursor"] = self.state["new_last_cursor"]
|
||||
self.state["last_timestamp_us"] = self.state["new_last_timestamp_us"]
|
||||
self.state["sent_cursors"] = self.state["new_sent_cursors"]
|
||||
self.state["last_run_timestamp"] = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
return logs
|
||||
|
||||
def _collect_syslog_fallback(self, hours: int = 24) -> List[Dict[str, Any]]:
|
||||
"""Fallback log reader for non-systemd Linux environments."""
|
||||
candidates = ["/var/log/syslog", "/var/log/messages"]
|
||||
target = next((f for f in candidates if os.path.exists(f) and os.access(f, os.R_OK)), None)
|
||||
if not target:
|
||||
return []
|
||||
|
||||
logs = []
|
||||
try:
|
||||
with open(target, "r", encoding="utf-8", errors="ignore") as f:
|
||||
lines = f.readlines()[-500:] # Read recent 500 lines
|
||||
for line in lines:
|
||||
line_str = line.strip()
|
||||
if not line_str:
|
||||
continue
|
||||
sev = "INFO"
|
||||
lower_l = line_str.lower()
|
||||
if any(w in lower_l for w in ["err", "crit", "alert", "emerg", "fail"]):
|
||||
sev = "ERROR"
|
||||
elif any(w in lower_l for w in ["warn", "notice"]):
|
||||
sev = "WARNING"
|
||||
|
||||
logs.append({
|
||||
"server": self.machine_id,
|
||||
"os_type": "linux",
|
||||
"signature": "syslog",
|
||||
"severity": sev,
|
||||
"message": line_str[:2048]
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
return logs
|
||||
|
||||
|
||||
def get_log_collector(
|
||||
os_type: Optional[str] = None,
|
||||
state: Optional[dict] = None,
|
||||
machine_id: Optional[str] = None,
|
||||
channels: Optional[List[str]] = None
|
||||
) -> BaseLogCollector:
|
||||
"""
|
||||
Factory helper that returns the appropriate BaseLogCollector instance.
|
||||
If os_type is omitted, auto-detects based on sys.platform.
|
||||
"""
|
||||
target_os = os_type.lower() if os_type else ("windows" if sys.platform == "win32" else "linux")
|
||||
if target_os == "windows":
|
||||
chans = channels or (["Application", "System"] if channels is None else ["Application"])
|
||||
return WindowsLogCollector(state=state, machine_id=machine_id, channels=chans)
|
||||
else:
|
||||
return LinuxLogCollector(state=state, machine_id=machine_id)
|
||||
from src.clients.log_collector import *
|
||||
@@ -0,0 +1,11 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
_CUR = os.path.dirname(__file__)
|
||||
_ROOT = os.path.abspath(os.path.join(_CUR, ".."))
|
||||
|
||||
for _p in [_CUR, _ROOT]:
|
||||
if _p and _p not in sys.path:
|
||||
sys.path.insert(0, _p)
|
||||
|
||||
from src.hub.pki_manager import *
|
||||
+7
-1015
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,11 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
_CUR = os.path.dirname(__file__)
|
||||
_ROOT = os.path.abspath(os.path.join(_CUR, ".."))
|
||||
|
||||
for _p in [_CUR, _ROOT]:
|
||||
if _p and _p not in sys.path:
|
||||
sys.path.insert(0, _p)
|
||||
|
||||
from src.hub.server_models import *
|
||||
@@ -0,0 +1,11 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
_CUR = os.path.dirname(__file__)
|
||||
_ROOT = os.path.abspath(os.path.join(_CUR, ".."))
|
||||
|
||||
for _p in [_CUR, _ROOT]:
|
||||
if _p and _p not in sys.path:
|
||||
sys.path.insert(0, _p)
|
||||
|
||||
from src.hub.storage_engine import *
|
||||
@@ -0,0 +1,11 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
_CUR = os.path.dirname(__file__)
|
||||
_ROOT = os.path.abspath(os.path.join(_CUR, ".."))
|
||||
|
||||
for _p in [_CUR, _ROOT]:
|
||||
if _p and _p not in sys.path:
|
||||
sys.path.insert(0, _p)
|
||||
|
||||
from src.hub.update_manager import *
|
||||
@@ -0,0 +1,11 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
_CUR = os.path.dirname(__file__)
|
||||
_ROOT = os.path.abspath(os.path.join(_CUR, ".."))
|
||||
|
||||
for _p in [_CUR, _ROOT]:
|
||||
if _p and _p not in sys.path:
|
||||
sys.path.insert(0, _p)
|
||||
|
||||
from src.common.version import *
|
||||
@@ -0,0 +1,11 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
_CUR = os.path.dirname(__file__)
|
||||
_ROOT = os.path.abspath(os.path.join(_CUR, ".."))
|
||||
|
||||
for _p in [_CUR, _ROOT]:
|
||||
if _p and _p not in sys.path:
|
||||
sys.path.insert(0, _p)
|
||||
|
||||
from src.hub.web_ui import *
|
||||
@@ -0,0 +1,178 @@
|
||||
"""
|
||||
LOGAR - Comprehensive Unit Tests for Edge Clients & Forwarders
|
||||
Covers log_collector, client_transport, client_updater, Win_Client, and Linux_Client.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import tempfile
|
||||
import unittest
|
||||
import socket
|
||||
import ssl
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from src.clients.log_collector import (
|
||||
get_machine_identifier,
|
||||
is_path_writable,
|
||||
get_default_data_dir,
|
||||
resolve_writable_path,
|
||||
get_state_path,
|
||||
load_state,
|
||||
save_state,
|
||||
commit_state,
|
||||
WindowsLogCollector,
|
||||
LinuxLogCollector,
|
||||
BaseLogCollector,
|
||||
get_log_collector,
|
||||
is_routine_service_noise,
|
||||
collect_system_metrics
|
||||
)
|
||||
from src.clients.client_updater import ClientUpdater
|
||||
from src.clients.client_transport import (
|
||||
is_cert_expiring_soon,
|
||||
enroll_client_if_needed,
|
||||
load_config,
|
||||
ensure_enrolled,
|
||||
send_encrypted_logs_over_socket,
|
||||
check_and_apply_auto_update
|
||||
)
|
||||
import src.clients.Win_Client as Win_Client
|
||||
import src.clients.Linux_Client as Linux_Client
|
||||
|
||||
|
||||
class TestClientsComprehensive(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
|
||||
def tearDown(self):
|
||||
import shutil
|
||||
if os.path.exists(self.test_dir):
|
||||
try:
|
||||
shutil.rmtree(self.test_dir)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def test_log_collector_utilities(self):
|
||||
"""Tests machine identifier, path resolution, and state persistence."""
|
||||
mid = get_machine_identifier()
|
||||
self.assertTrue(len(mid) > 0)
|
||||
|
||||
data_dir = get_default_data_dir("TEST-LOGAR")
|
||||
self.assertTrue(isinstance(data_dir, str))
|
||||
|
||||
resolved = resolve_writable_path(os.path.join(self.test_dir, "test.db"), "TEST-LOGAR")
|
||||
self.assertEqual(os.path.abspath(resolved), os.path.abspath(os.path.join(self.test_dir, "test.db")))
|
||||
|
||||
state_path = get_state_path("test_state.json", "TEST-LOGAR")
|
||||
self.assertTrue(isinstance(state_path, str))
|
||||
|
||||
# State load/save/commit
|
||||
st_file = os.path.join(self.test_dir, "st.json")
|
||||
st_data = load_state(st_file)
|
||||
self.assertIsInstance(st_data, dict)
|
||||
st_data["processed_records"] = 42
|
||||
st_data["new_last_record_number"] = 100
|
||||
save_state(st_file, st_data)
|
||||
loaded = load_state(st_file)
|
||||
self.assertEqual(loaded["processed_records"], 42)
|
||||
|
||||
commit_state(loaded, st_file)
|
||||
loaded2 = load_state(st_file)
|
||||
self.assertEqual(loaded2["last_record_number"], 100)
|
||||
|
||||
def test_metrics_collection_and_noise_filter(self):
|
||||
"""Tests system metrics telemetry and routine noise filtering."""
|
||||
metrics = collect_system_metrics()
|
||||
self.assertIn("cpu_percent", metrics)
|
||||
self.assertIn("memory_percent", metrics)
|
||||
self.assertIn("disk_percent_free", metrics)
|
||||
|
||||
# Routine noise filter: signature, message, severity
|
||||
self.assertTrue(is_routine_service_noise("Service Control Manager", "Windows Error Reporting Service entered the stopped state.", "INFO"))
|
||||
self.assertTrue(is_routine_service_noise("Service Control Manager", "Background Intelligent Transfer Service entered the running state.", "INFO"))
|
||||
self.assertFalse(is_routine_service_noise("NTFS", "Fatal disk corruption error detected on C:", "ERROR"))
|
||||
|
||||
def test_log_collector_classes(self):
|
||||
"""Tests WindowsLogCollector and LinuxLogCollector implementations."""
|
||||
# Linux Log Collector
|
||||
linux_col = LinuxLogCollector()
|
||||
self.assertIsNotNone(linux_col)
|
||||
|
||||
# Windows Log Collector
|
||||
win_col = WindowsLogCollector()
|
||||
self.assertEqual(win_col.channels, ["Application"])
|
||||
|
||||
factory_col = get_log_collector()
|
||||
self.assertIsInstance(factory_col, BaseLogCollector)
|
||||
|
||||
def test_client_updater_lifecycle(self):
|
||||
"""Tests ClientUpdater download, local hash match, and update application routines."""
|
||||
updater = ClientUpdater(config={"server_host": "127.0.0.1", "hermes_port": 8443}, os_type="linux")
|
||||
self.assertEqual(updater.os_type, "linux")
|
||||
base_url = updater.get_server_base_url()
|
||||
self.assertEqual(base_url, "http://127.0.0.1:8443")
|
||||
|
||||
# Linux apply update mock
|
||||
dummy_bin = os.path.join(self.test_dir, "dummy_new_bin")
|
||||
dummy_target = os.path.join(self.test_dir, "dummy_target_bin")
|
||||
with open(dummy_bin, "wb") as f:
|
||||
f.write(b"NEW_BINARY_CONTENT")
|
||||
with open(dummy_target, "wb") as f:
|
||||
f.write(b"OLD_BINARY_CONTENT")
|
||||
|
||||
ok = updater.apply_update_linux(dummy_bin, dummy_target)
|
||||
self.assertTrue(ok)
|
||||
with open(dummy_target, "rb") as f:
|
||||
self.assertEqual(f.read(), b"NEW_BINARY_CONTENT")
|
||||
|
||||
# Windows updater batch generation
|
||||
updater_win = ClientUpdater(config={"server_host": "127.0.0.1", "hermes_port": 8443}, os_type="windows")
|
||||
with patch("subprocess.Popen") as mock_popen:
|
||||
win_bin = os.path.join(self.test_dir, "win_new.exe")
|
||||
win_target = os.path.join(self.test_dir, "win_target.exe")
|
||||
with open(win_bin, "wb") as f:
|
||||
f.write(b"WIN_EXE")
|
||||
with open(win_target, "wb") as f:
|
||||
f.write(b"WIN_TARGET")
|
||||
res = updater_win.apply_update_windows(win_bin, win_target)
|
||||
self.assertTrue(res)
|
||||
self.assertTrue(mock_popen.called)
|
||||
|
||||
def test_client_transport_config_and_enrollment(self):
|
||||
"""Tests loading client config, cert expiration check, and fallback."""
|
||||
cfg_file = os.path.join(self.test_dir, "client_config.json")
|
||||
cert_dir = os.path.join(self.test_dir, "certs")
|
||||
os.makedirs(cert_dir, exist_ok=True)
|
||||
cfg_data = {
|
||||
"server_host": "127.0.0.1",
|
||||
"server_port": 9443,
|
||||
"hermes_host": "127.0.0.1",
|
||||
"hermes_port": 8443,
|
||||
"enrollment_secret": "test_sec",
|
||||
"cert_dir": cert_dir,
|
||||
"server_fingerprint": "FP123",
|
||||
"server_public_key": "",
|
||||
"auth_token": "token123"
|
||||
}
|
||||
with open(cfg_file, "w", encoding="utf-8") as f:
|
||||
json.dump(cfg_data, f)
|
||||
|
||||
loaded_cfg = load_config(cfg_file)
|
||||
self.assertEqual(loaded_cfg["server_host"], "127.0.0.1")
|
||||
|
||||
# Expiry check
|
||||
self.assertTrue(is_cert_expiring_soon(os.path.join(cert_dir, "nonexistent.crt")))
|
||||
|
||||
def test_client_main_helpers(self):
|
||||
"""Tests Win_Client and Linux_Client config loading and log retrieval fallbacks."""
|
||||
logs = Win_Client.get_recent_windows_logs(hours=1, state={})
|
||||
self.assertIsInstance(logs, list)
|
||||
|
||||
logs_l = Linux_Client.get_recent_linux_logs(hours=1, state={})
|
||||
self.assertIsInstance(logs_l, list)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,109 @@
|
||||
"""
|
||||
Comprehensive unit tests for LOGAR Client Transport, Windows & Linux forwarders, and update checks.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from src.clients import client_transport
|
||||
from src.clients import Win_Client, Linux_Client
|
||||
|
||||
|
||||
class TestClientsDeep(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.cert_dir = os.path.join(self.temp_dir, "certs")
|
||||
os.makedirs(self.cert_dir, exist_ok=True)
|
||||
self.config_path = os.path.join(self.temp_dir, "client_config.json")
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.temp_dir, ignore_errors=True)
|
||||
|
||||
def test_client_transport_config_loader_and_helpers(self):
|
||||
"""Tests client configuration loading, writable directory fallback, and path resolution."""
|
||||
sample_config = {
|
||||
"server_host": "127.0.0.1",
|
||||
"server_port": 9443,
|
||||
"hermes_port": 8443,
|
||||
"cert_dir": self.cert_dir,
|
||||
"enrollment_secret": "test_enroll_secret_2026",
|
||||
"auth_token": "token_123",
|
||||
"ca_cert": "-----BEGIN CERTIFICATE-----\nMIIB...\n-----END CERTIFICATE-----\n"
|
||||
}
|
||||
with open(self.config_path, "w", encoding="utf-8") as f:
|
||||
json.dump(sample_config, f)
|
||||
|
||||
cfg = client_transport.load_config(self.config_path)
|
||||
self.assertEqual(cfg["server_host"], "127.0.0.1")
|
||||
self.assertEqual(cfg["server_port"], 9443)
|
||||
self.assertTrue(os.path.exists(os.path.join(cfg["cert_dir"], "ca.crt")))
|
||||
|
||||
# Test cert expiry check on non-existent cert
|
||||
self.assertTrue(client_transport.is_cert_expiring_soon(os.path.join(self.cert_dir, "nonexistent.crt")))
|
||||
|
||||
def test_client_transport_ensure_enrolled(self):
|
||||
"""Tests proactive client enrollment bootstrap check."""
|
||||
cfg = {
|
||||
"server_host": "127.0.0.1",
|
||||
"server_port": 9443,
|
||||
"hermes_host": "127.0.0.1",
|
||||
"hermes_port": 8443,
|
||||
"cert_dir": self.cert_dir,
|
||||
"enrollment_secret": "secret"
|
||||
}
|
||||
|
||||
with patch("src.clients.client_transport.enroll_client_if_needed") as mock_enroll:
|
||||
mock_enroll.return_value = True
|
||||
ok = client_transport.ensure_enrolled(cfg, "machine_01", os_type="windows")
|
||||
self.assertTrue(ok)
|
||||
mock_enroll.assert_called_once()
|
||||
|
||||
def test_client_transport_openpgp_fallback_transmission(self):
|
||||
"""Tests fallback to OpenPGP envelope over plain TCP socket when certs are absent."""
|
||||
from src.hub.storage_engine import StorageCryptoManager
|
||||
priv, pub, fp = StorageCryptoManager.generate_server_keypair("Hub Test Key")
|
||||
|
||||
cfg = {
|
||||
"server_host": "127.0.0.1",
|
||||
"server_port": 9443,
|
||||
"cert_dir": os.path.join(self.temp_dir, "empty_certs"),
|
||||
"server_public_key": pub,
|
||||
"server_fingerprint": fp,
|
||||
"auth_token": "client_token_abc"
|
||||
}
|
||||
|
||||
mock_socket = MagicMock()
|
||||
# Mock server returning 4-byte big-endian length + JSON response
|
||||
resp_data = json.dumps({"status": "success", "processed": 5}).encode("utf-8")
|
||||
resp_len_header = len(resp_data).to_bytes(4, byteorder="big")
|
||||
|
||||
mock_socket.recv.side_effect = [resp_len_header, resp_data, b""]
|
||||
|
||||
with patch("socket.socket") as mock_sock_cls:
|
||||
mock_sock_cls.return_value.__enter__.return_value = mock_socket
|
||||
logs = [{"timestamp": "2026-01-01T00:00:00Z", "event_id": 4625, "message": "Failed logon"}]
|
||||
res = client_transport.send_encrypted_logs_over_socket(cfg, logs, machine_id="test_win_node")
|
||||
self.assertEqual(res.get("status"), "success")
|
||||
self.assertEqual(res.get("processed"), 5)
|
||||
|
||||
def test_win_client_and_linux_client_entrypoints(self):
|
||||
"""Tests command-line parsing, help options, and diagnostic checks."""
|
||||
# Test help flag execution
|
||||
with self.assertRaises(SystemExit) as cm:
|
||||
with patch("sys.argv", ["Win_Client.py", "--help"]):
|
||||
Win_Client.main()
|
||||
self.assertEqual(cm.exception.code, 0)
|
||||
|
||||
with self.assertRaises(SystemExit) as cm:
|
||||
with patch("sys.argv", ["Linux_Client.py", "--help"]):
|
||||
Linux_Client.main()
|
||||
self.assertEqual(cm.exception.code, 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,52 @@
|
||||
"""
|
||||
LOGAR - Comprehensive Unit Tests for Common Version Module
|
||||
100% test coverage for semantic versioning, version parsing, comparisons, and edge cases.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from src.common.version import (
|
||||
LOGAR_VERSION,
|
||||
parse_version,
|
||||
compare_versions,
|
||||
is_newer_version
|
||||
)
|
||||
|
||||
|
||||
class TestCommonVersion(unittest.TestCase):
|
||||
def test_version_constants(self):
|
||||
self.assertIsInstance(LOGAR_VERSION, str)
|
||||
self.assertTrue(len(LOGAR_VERSION) > 0)
|
||||
|
||||
def test_parse_version_valid(self):
|
||||
self.assertEqual(parse_version("1.2.3.4"), (1, 2, 3, 4))
|
||||
self.assertEqual(parse_version("v1.2.3.4"), (1, 2, 3, 4))
|
||||
self.assertEqual(parse_version("V1.2.3"), (1, 2, 3, 0))
|
||||
self.assertEqual(parse_version("1.2"), (1, 2, 0, 0))
|
||||
self.assertEqual(parse_version("1"), (1, 0, 0, 0))
|
||||
self.assertEqual(parse_version("1.2.3.4.5"), (1, 2, 3, 4, 5))
|
||||
|
||||
def test_parse_version_invalid_and_edge_cases(self):
|
||||
self.assertEqual(parse_version(None), (0, 0, 0, 0))
|
||||
self.assertEqual(parse_version(""), (0, 0, 0, 0))
|
||||
self.assertEqual(parse_version(" "), (0, 0, 0, 0))
|
||||
self.assertEqual(parse_version("invalid"), (0, 0, 0, 0))
|
||||
self.assertEqual(parse_version("v1.alpha.2"), (1, 0, 0, 0))
|
||||
|
||||
def test_compare_versions(self):
|
||||
self.assertEqual(compare_versions("1.0.0.0", "1.0.0.0"), 0)
|
||||
self.assertEqual(compare_versions("1.0.0.1", "1.0.0.0"), 1)
|
||||
self.assertEqual(compare_versions("1.0.0.0", "1.0.0.1"), -1)
|
||||
self.assertEqual(compare_versions("2.0.0.0", "1.9.9.9"), 1)
|
||||
self.assertEqual(compare_versions("0.3.5.2", "0.3.5.2"), 0)
|
||||
self.assertEqual(compare_versions("0.3.6.0", "0.3.5.2"), 1)
|
||||
self.assertEqual(compare_versions("0.3.5.1", "0.3.5.2"), -1)
|
||||
|
||||
def test_is_newer_version(self):
|
||||
self.assertTrue(is_newer_version("1.0.0.1", "1.0.0.0"))
|
||||
self.assertFalse(is_newer_version("1.0.0.0", "1.0.0.0"))
|
||||
self.assertFalse(is_newer_version("0.9.9.9", "1.0.0.0"))
|
||||
self.assertTrue(is_newer_version("v0.3.6.0", "0.3.5.2"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,201 @@
|
||||
"""
|
||||
LOGAR - Comprehensive Unit Tests for Central Server Hub
|
||||
Covers License_Authority, auth_manager, ingest_queue, license_manager, pki_manager,
|
||||
server_enrollment, server_models, storage_engine, update_manager, web_ui, and Server endpoints.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import uuid
|
||||
import time
|
||||
import tempfile
|
||||
import unittest
|
||||
import sqlite3
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from src.hub.server_models import (
|
||||
ClientEnrollRequest,
|
||||
ClientActionRequest,
|
||||
ClientUpdateRequest,
|
||||
LicenseImportRequest,
|
||||
HermesTokenCreateRequest,
|
||||
HermesTokenRevokeRequest,
|
||||
ClientProfileExportRequest,
|
||||
ChangePasswordRequest,
|
||||
DbConnectionTestRequest,
|
||||
DbSwitchBackendRequest,
|
||||
DbFullConfigRequest
|
||||
)
|
||||
from src.hub.auth_manager import (
|
||||
PasswordManager,
|
||||
SessionManager,
|
||||
RateLimiter,
|
||||
TOTPManager,
|
||||
HermesTokenManager
|
||||
)
|
||||
from src.hub.ingest_queue import IngestQueueManager
|
||||
from src.hub.license_manager import LicenseManager
|
||||
from src.hub.pki_manager import PKIManager
|
||||
from src.hub.storage_engine import StorageEngine
|
||||
from src.hub.update_manager import ServerUpdateManager
|
||||
from src.hub.web_ui import WebUIRenderer
|
||||
import src.hub.Server as Server
|
||||
import src.hub.server_enrollment as enrollment
|
||||
|
||||
|
||||
class TestHubComprehensive(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
self.test_db = os.path.join(self.test_dir, f"test_hub_{uuid.uuid4().hex[:8]}.db")
|
||||
self.cert_dir = os.path.join(self.test_dir, "certs")
|
||||
self.updates_dir = os.path.join(self.test_dir, "updates")
|
||||
|
||||
StorageEngine.init_db(self.test_db, max_seats=10)
|
||||
self.update_mgr = ServerUpdateManager(update_dir=self.updates_dir, check_enabled=False)
|
||||
Server.SERVER_STATE["update_manager"] = self.update_mgr
|
||||
Server.SERVER_STATE["config"] = {
|
||||
"db_path": self.test_db,
|
||||
"cert_dir": self.cert_dir,
|
||||
"server_name": "LOGAR-Test-Hub",
|
||||
"server_fingerprint": "TEST_FP_123",
|
||||
"tcp_host": "127.0.0.1",
|
||||
"tcp_port": 9443,
|
||||
"hermes_port": 8443,
|
||||
"auth_token": "test_auth_token_secret"
|
||||
}
|
||||
|
||||
def tearDown(self):
|
||||
import shutil
|
||||
if os.path.exists(self.test_dir):
|
||||
try:
|
||||
shutil.rmtree(self.test_dir)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def test_server_models(self):
|
||||
"""Tests Pydantic validation on all Server Hub data models."""
|
||||
enr = ClientEnrollRequest(client_id="c1", hostname="host1", os="windows", enrollment_secret="sec")
|
||||
self.assertEqual(enr.client_id, "c1")
|
||||
|
||||
act = ClientActionRequest(client_id="c1")
|
||||
self.assertEqual(act.client_id, "c1")
|
||||
|
||||
upd = ClientUpdateRequest(client_id="c1", os="windows", current_version="0.3.5.0")
|
||||
self.assertEqual(upd.current_version, "0.3.5.0")
|
||||
|
||||
lic_imp = LicenseImportRequest(ca_cert="PEM_CERT_DATA")
|
||||
self.assertEqual(lic_imp.ca_cert, "PEM_CERT_DATA")
|
||||
|
||||
htc = HermesTokenCreateRequest(name="agent1")
|
||||
self.assertEqual(htc.name, "agent1")
|
||||
|
||||
htr = HermesTokenRevokeRequest(token="tok123")
|
||||
self.assertEqual(htr.token, "tok123")
|
||||
|
||||
cpe = ClientProfileExportRequest(server_host="10.0.0.1")
|
||||
self.assertEqual(cpe.server_host, "10.0.0.1")
|
||||
|
||||
cp = ChangePasswordRequest(old_password="old", new_password="new")
|
||||
self.assertEqual(cp.new_password, "new")
|
||||
|
||||
dct = DbConnectionTestRequest(host="localhost", port=5432, database="logar")
|
||||
self.assertEqual(dct.port, 5432)
|
||||
|
||||
dfc = DbFullConfigRequest(storage_engine="sqlite", cache_mode="direct")
|
||||
self.assertEqual(dfc.storage_engine, "sqlite")
|
||||
|
||||
def test_auth_manager_edge_cases(self):
|
||||
"""Tests password hashing, TOTP secret generation, and rate limiting edge cases."""
|
||||
# Password
|
||||
h, s = PasswordManager.hash_password("admin_password")
|
||||
self.assertTrue(PasswordManager.verify_password("admin_password", s, h))
|
||||
self.assertFalse(PasswordManager.verify_password("wrong_password", s, h))
|
||||
|
||||
# Session token
|
||||
tok = SessionManager.generate_session_token("admin", db_path=self.test_db)
|
||||
self.assertEqual(SessionManager.verify_session_token(tok, db_path=self.test_db), "admin")
|
||||
self.assertIsNone(SessionManager.verify_session_token("invalid_token", db_path=self.test_db))
|
||||
|
||||
# TOTP Manager
|
||||
sec = TOTPManager.generate_totp_secret(length=20)
|
||||
self.assertTrue(len(sec) >= 20)
|
||||
code = TOTPManager.get_totp_token(sec)
|
||||
self.assertTrue(len(code) == 6)
|
||||
self.assertTrue(TOTPManager.verify_totp_code(sec, code))
|
||||
uri = TOTPManager.get_totp_uri("admin", sec)
|
||||
self.assertIn("otpauth://totp/", uri)
|
||||
|
||||
# Hermes Token Manager
|
||||
ht = HermesTokenManager.create_hermes_token("Agent 1", db_path=self.test_db)
|
||||
self.assertIn("token", ht)
|
||||
self.assertTrue(HermesTokenManager.verify_hermes_token(ht["token"], db_path=self.test_db))
|
||||
tokens = HermesTokenManager.list_hermes_tokens(db_path=self.test_db)
|
||||
self.assertTrue(len(tokens) >= 1)
|
||||
HermesTokenManager.revoke_hermes_token(ht["token"], db_path=self.test_db)
|
||||
self.assertFalse(HermesTokenManager.verify_hermes_token(ht["token"], db_path=self.test_db))
|
||||
HermesTokenManager.delete_hermes_token(ht["token"], db_path=self.test_db)
|
||||
|
||||
def test_pki_and_server_enrollment(self):
|
||||
"""Tests dynamic CA, Server, and client certificate issuance and verification."""
|
||||
ca_cert, ca_key, ca_pem, ca_key_pem = enrollment.generate_ca_if_needed(cert_dir=self.cert_dir)
|
||||
self.assertIsNotNone(ca_cert)
|
||||
self.assertTrue(os.path.exists(os.path.join(self.cert_dir, "ca.crt")))
|
||||
|
||||
# Issue server cert
|
||||
srv_cert, srv_key, srv_pem, srv_key_pem = enrollment.generate_server_cert_if_needed(
|
||||
ca_cert, ca_key,
|
||||
hostnames=["127.0.0.1", "localhost"],
|
||||
cert_dir=self.cert_dir
|
||||
)
|
||||
self.assertIsNotNone(srv_cert)
|
||||
|
||||
# Issue client cert
|
||||
cl_pem, cl_key_pem = enrollment.issue_client_cert(
|
||||
client_id="client-hub-test",
|
||||
ca_cert=ca_cert,
|
||||
ca_key=ca_key
|
||||
)
|
||||
self.assertIn("BEGIN CERTIFICATE", cl_pem)
|
||||
self.assertIn("BEGIN RSA PRIVATE KEY", cl_key_pem)
|
||||
|
||||
# Validate renewal logic
|
||||
ca_ren, srv_ren = enrollment.check_and_renew_hub_pki(cert_dir=self.cert_dir, hostnames=["127.0.0.1"])
|
||||
self.assertFalse(ca_ren)
|
||||
self.assertFalse(srv_ren)
|
||||
rotated = PKIManager.check_and_rotate_server_certs(cert_dir=self.cert_dir, hostnames=["127.0.0.1"])
|
||||
self.assertFalse(rotated)
|
||||
|
||||
def test_storage_engine_sqlite_and_migrations(self):
|
||||
"""Tests StorageEngine table initialization, column migrations, and data persistence."""
|
||||
db_path = StorageEngine.init_db(self.test_db)
|
||||
self.assertTrue(os.path.exists(db_path))
|
||||
|
||||
# Check tables
|
||||
conn = sqlite3.connect(db_path)
|
||||
tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()}
|
||||
conn.close()
|
||||
|
||||
self.assertIn("active_issues", tables)
|
||||
self.assertIn("clients", tables)
|
||||
self.assertIn("license_config", tables)
|
||||
self.assertIn("users", tables)
|
||||
self.assertIn("hermes_tokens", tables)
|
||||
|
||||
def test_web_ui_renderer_all_tabs(self):
|
||||
"""Tests rendering of all WebUI tabs: Fleet, Hermes, Anomalies, PKI, DB, Updates."""
|
||||
html = WebUIRenderer.render_dashboard_html(
|
||||
server_state=Server.SERVER_STATE,
|
||||
ingest_queue=Server.INGEST_QUEUE
|
||||
)
|
||||
self.assertIn("LOGAR Fleet Hub", html)
|
||||
self.assertIn("tabPane-fleet", html)
|
||||
self.assertIn("tabPane-hermes", html)
|
||||
self.assertIn("tabPane-anomalies", html)
|
||||
self.assertIn("tabPane-pki", html)
|
||||
self.assertIn("tabPane-db", html)
|
||||
self.assertIn("tabPane-updates", html)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,100 @@
|
||||
"""
|
||||
Comprehensive unit tests for dynamic PKI, Sub-CA issuance, offline CSR activation, and LicenseAuthority.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import sqlite3
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from src.hub import server_enrollment as enrollment
|
||||
from src.hub.pki_manager import PKIManager
|
||||
|
||||
|
||||
class TestHubEnrollmentPKIDeep(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.cert_dir = os.path.join(self.temp_dir, "certs")
|
||||
os.makedirs(self.cert_dir, exist_ok=True)
|
||||
self.db_path = os.path.join(self.temp_dir, "logar_test.db")
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.temp_dir, ignore_errors=True)
|
||||
|
||||
def test_pki_lifecycle_and_extensions(self):
|
||||
"""Tests CA generation, Server TLS cert generation, client cert issuance, and license OID parsing."""
|
||||
# 1. CA Generation
|
||||
ca_cert, ca_key, ca_pem, ca_key_pem = enrollment.generate_ca_if_needed(cert_dir=self.cert_dir)
|
||||
self.assertIsNotNone(ca_cert)
|
||||
self.assertTrue(os.path.exists(os.path.join(self.cert_dir, "ca.crt")))
|
||||
self.assertTrue(os.path.exists(os.path.join(self.cert_dir, "ca.key")))
|
||||
|
||||
# 2. Server TLS Certificate Generation
|
||||
srv_cert, srv_key, srv_pem, srv_key_pem = enrollment.generate_server_cert_if_needed(
|
||||
ca_cert=ca_cert,
|
||||
ca_key=ca_key,
|
||||
hostnames=["127.0.0.1", "localhost"],
|
||||
cert_dir=self.cert_dir
|
||||
)
|
||||
self.assertIsNotNone(srv_cert)
|
||||
self.assertTrue(os.path.exists(os.path.join(self.cert_dir, "server.crt")))
|
||||
self.assertTrue(os.path.exists(os.path.join(self.cert_dir, "server.key")))
|
||||
|
||||
# 3. Client Certificate Issuance
|
||||
cl_pem, cl_key_pem = enrollment.issue_client_cert(
|
||||
client_id="forwarder-node-01",
|
||||
ca_cert=ca_cert,
|
||||
ca_key=ca_key,
|
||||
days_valid=365
|
||||
)
|
||||
self.assertIn("BEGIN CERTIFICATE", cl_pem)
|
||||
self.assertIn("BEGIN RSA PRIVATE KEY", cl_key_pem)
|
||||
|
||||
# 4. Extract license info
|
||||
info = enrollment.extract_license_info_from_cert(ca_pem)
|
||||
self.assertIn("is_self_signed", info)
|
||||
|
||||
# 5. Check cert expiry helper
|
||||
self.assertFalse(enrollment.is_cert_expiring_soon(ca_pem, threshold_days=30))
|
||||
|
||||
# 6. Initialize mTLS Server context
|
||||
ctx = PKIManager.init_mtls_server_context(cert_dir=self.cert_dir)
|
||||
self.assertIsNotNone(ctx)
|
||||
|
||||
# 7. Reload mTLS context
|
||||
PKIManager.reload_mtls_context(ctx, cert_dir=self.cert_dir)
|
||||
|
||||
def test_offline_csr_and_activation(self):
|
||||
"""Tests export of offline license request CSR and activation bundle unpack."""
|
||||
csr_pem, csr_key_pem = enrollment.generate_ca_csr(cert_dir=self.cert_dir)
|
||||
self.assertIn("BEGIN CERTIFICATE REQUEST", csr_pem)
|
||||
|
||||
# Sign the CSR locally as Sub-CA for test
|
||||
ca_cert, ca_key, ca_pem, ca_key_pem = enrollment.generate_ca_if_needed(cert_dir=self.cert_dir)
|
||||
sub_ca_pem = enrollment.sign_ca_csr_with_seats(
|
||||
csr_pem=csr_pem,
|
||||
master_ca_cert_pem=ca_pem,
|
||||
master_ca_key_pem=ca_key_pem,
|
||||
seats=25,
|
||||
days_valid=365,
|
||||
license_id="LIC-TEST-2026",
|
||||
features=["postgres", "realtime_alerting"]
|
||||
)
|
||||
self.assertIn("BEGIN CERTIFICATE", sub_ca_pem)
|
||||
|
||||
# Verify signed Sub-CA contains license extensions
|
||||
info = enrollment.extract_license_info_from_cert(sub_ca_pem)
|
||||
self.assertEqual(info.get("seats"), 25)
|
||||
self.assertEqual(info.get("license_id"), "LIC-TEST-2026")
|
||||
self.assertIn("postgres", info.get("features", []))
|
||||
|
||||
# Test create_license_request_lic
|
||||
lic_data, lic_json = enrollment.create_license_request_lic(cert_dir=self.cert_dir)
|
||||
self.assertEqual(lic_data["format"], "LOGAR_LICENSE_REQUEST")
|
||||
self.assertIn("csr", lic_data)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,150 @@
|
||||
"""
|
||||
Comprehensive unit tests for LOGAR Server FastAPI endpoints, state management, and log processing.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
import sqlite3
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from src.hub import Server
|
||||
from src.hub.server_models import (
|
||||
ClientEnrollRequest,
|
||||
ClientActionRequest,
|
||||
HermesTokenCreateRequest,
|
||||
HermesTokenRevokeRequest
|
||||
)
|
||||
from src.hub.auth_manager import SessionManager
|
||||
|
||||
|
||||
class TestHubServerDeep(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.cert_dir = os.path.join(self.temp_dir, "certs")
|
||||
os.makedirs(self.cert_dir, exist_ok=True)
|
||||
self.db_path = os.path.join(self.temp_dir, "test_server.db")
|
||||
self.config_path = os.path.join(self.temp_dir, "server_config.json")
|
||||
|
||||
# Initialize test state
|
||||
Server.StorageEngine.init_db(self.db_path, enrollment_secret="test_secret", max_seats=50)
|
||||
Server.SERVER_STATE["config"] = {
|
||||
"db_path": self.db_path,
|
||||
"enrollment_secret": "test_secret",
|
||||
"max_seats": 50,
|
||||
"cert_dir": self.cert_dir,
|
||||
"encrypt_at_rest": False
|
||||
}
|
||||
# Generate and load CA
|
||||
ca_cert, ca_key, ca_pem, _ = Server.enrollment.generate_ca_if_needed(cert_dir=self.cert_dir)
|
||||
Server.SERVER_STATE["ca_cert"] = ca_cert
|
||||
Server.SERVER_STATE["ca_key"] = ca_key
|
||||
Server.SERVER_STATE["ca_cert_pem"] = ca_pem
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.temp_dir, ignore_errors=True)
|
||||
|
||||
def test_server_status_and_update_routes(self):
|
||||
"""Tests /api/update/status and health check handler."""
|
||||
status_data = Server.get_update_status(user="admin")
|
||||
self.assertIn("hosted_packages", status_data)
|
||||
|
||||
health = Server.health_check()
|
||||
self.assertEqual(health.get("status"), "healthy")
|
||||
|
||||
def test_client_enrollment_and_action_endpoints(self):
|
||||
"""Tests client enrollment, listing, revoke, approve, and delete handlers."""
|
||||
# Ensure license in DB is marked active with seats
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
conn.execute("UPDATE license_config SET max_seats = 50, license_status = 'active' WHERE id = 1")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
# Enroll client
|
||||
enroll_req = ClientEnrollRequest(
|
||||
client_id="test_agent_deep_01",
|
||||
hostname="workstation-01",
|
||||
os="windows",
|
||||
enrollment_secret="test_secret"
|
||||
)
|
||||
enroll_data = Server.enroll_client(enroll_req)
|
||||
self.assertEqual(enroll_data.get("status"), "success")
|
||||
self.assertIn("client_cert", enroll_data)
|
||||
self.assertIn("client_key", enroll_data)
|
||||
|
||||
# Check client list
|
||||
clients_data = Server.list_clients(auth_user="admin")
|
||||
self.assertGreaterEqual(clients_data.get("active_seats", 0), 1)
|
||||
|
||||
# Client action: revoke
|
||||
action_req = ClientActionRequest(client_id="test_agent_deep_01")
|
||||
rev_data = Server.revoke_client(action_req, auth_user="admin")
|
||||
self.assertEqual(rev_data.get("status"), "revoked")
|
||||
|
||||
# Reactivate
|
||||
app_data = Server.approve_client(action_req, auth_user="admin")
|
||||
self.assertEqual(app_data.get("status"), "active")
|
||||
|
||||
# Delete
|
||||
del_data = Server.delete_client(action_req, auth_user="admin")
|
||||
self.assertEqual(del_data.get("status"), "deleted")
|
||||
|
||||
def test_hermes_tokens_and_auth_routes(self):
|
||||
"""Tests Hermes API token creation, listing, and revoking handlers."""
|
||||
# Create token
|
||||
tok_req = HermesTokenCreateRequest(name="Test Integration Agent")
|
||||
tok_data = Server.api_create_hermes_token(tok_req, auth_user="admin")
|
||||
token_val = tok_data.get("token")
|
||||
self.assertIsNotNone(token_val)
|
||||
|
||||
# List tokens
|
||||
tokens_list = Server.api_list_hermes_tokens(auth_user="admin")
|
||||
self.assertTrue(any(t.get("token") == token_val for t in tokens_list))
|
||||
|
||||
# Revoke token
|
||||
rev_req = HermesTokenRevokeRequest(token=token_val)
|
||||
rev_data = Server.api_revoke_hermes_token(rev_req, auth_user="admin")
|
||||
self.assertEqual(rev_data.get("status"), "success")
|
||||
|
||||
def test_process_ingested_logs_and_temporal_tracking(self):
|
||||
"""Tests processing ingested log entries, deduplication, and anomaly promotion."""
|
||||
payload = {
|
||||
"server": "web-prod-01",
|
||||
"timestamp": "2026-01-01T12:00:00Z",
|
||||
"logs": [
|
||||
{
|
||||
"site_name": "Default-Site",
|
||||
"server": "web-prod-01",
|
||||
"signature": "SECURITY_4625_BRUTEFORCE",
|
||||
"severity": "CRITICAL",
|
||||
"message": "Repeated failed logon attempts detected from 10.0.0.99",
|
||||
"os_type": "windows",
|
||||
"timestamp": "2026-01-01T12:00:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
res = Server.process_ingested_logs(
|
||||
payload=payload,
|
||||
db_path=self.db_path,
|
||||
window_hours=24,
|
||||
min_runs=1
|
||||
)
|
||||
self.assertEqual(res.get("processed"), 1)
|
||||
|
||||
# Query database directly to confirm issue recorded
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT signature, severity, run_count FROM active_issues")
|
||||
row = c.fetchone()
|
||||
conn.close()
|
||||
self.assertIsNotNone(row)
|
||||
self.assertEqual(row[0], "SECURITY_4625_BRUTEFORCE")
|
||||
self.assertEqual(row[1], "CRITICAL")
|
||||
self.assertEqual(row[2], 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,141 @@
|
||||
"""
|
||||
Comprehensive unit tests for LOGAR StorageEngine, StorageCryptoManager, and Multi-Backend Migrations.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
import sqlite3
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from src.hub.storage_engine import StorageEngine, StorageCryptoManager
|
||||
|
||||
|
||||
class TestStorageEngineDeep(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.db_path = os.path.join(self.temp_dir, "test_logar.db")
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.temp_dir, ignore_errors=True)
|
||||
|
||||
def test_storage_crypto_manager_keypair_and_encryption(self):
|
||||
"""Tests keypair generation, field encryption, decryption, and error resilience."""
|
||||
priv, pub, fp = StorageCryptoManager.generate_server_keypair("Test Server Key")
|
||||
self.assertIn("BEGIN PGP PRIVATE KEY", priv)
|
||||
self.assertIn("BEGIN PGP PUBLIC KEY", pub)
|
||||
self.assertTrue(len(fp) > 0)
|
||||
|
||||
server_state = {
|
||||
"config": {
|
||||
"encrypt_at_rest": True,
|
||||
"public_key": pub,
|
||||
"private_key": priv
|
||||
}
|
||||
}
|
||||
|
||||
# Test empty or non-string inputs
|
||||
self.assertEqual(StorageCryptoManager.encrypt_field_for_storage("", server_state=server_state, is_unlocked=True), "")
|
||||
self.assertEqual(StorageCryptoManager.encrypt_field_for_storage(None, server_state=server_state, is_unlocked=True), None)
|
||||
self.assertEqual(StorageCryptoManager.decrypt_field_from_storage("", server_state=server_state), "")
|
||||
self.assertEqual(StorageCryptoManager.decrypt_field_from_storage(None, server_state=server_state), None)
|
||||
|
||||
# Test non-encrypted plain string decrypt passthrough
|
||||
plain_msg = "Security Audit Message 2026"
|
||||
self.assertEqual(StorageCryptoManager.decrypt_field_from_storage(plain_msg, server_state=server_state), plain_msg)
|
||||
|
||||
# Encrypt when unlocked
|
||||
encrypted = StorageCryptoManager.encrypt_field_for_storage(plain_msg, server_state=server_state, is_unlocked=True)
|
||||
self.assertTrue(encrypted.startswith("-----BEGIN PGP MESSAGE-----"))
|
||||
|
||||
# Encrypt idempotence (should not double-encrypt)
|
||||
double_enc = StorageCryptoManager.encrypt_field_for_storage(encrypted, server_state=server_state, is_unlocked=True)
|
||||
self.assertEqual(double_enc, encrypted)
|
||||
|
||||
# Decrypt
|
||||
decrypted = StorageCryptoManager.decrypt_field_from_storage(encrypted, server_state=server_state)
|
||||
self.assertEqual(decrypted, plain_msg)
|
||||
|
||||
# Decrypt with state missing private_key_obj (reloads from blob)
|
||||
clean_state = {"config": {"private_key": priv}}
|
||||
decrypted_from_clean = StorageCryptoManager.decrypt_field_from_storage(encrypted, server_state=clean_state)
|
||||
self.assertEqual(decrypted_from_clean, plain_msg)
|
||||
|
||||
# Encrypt when locked / encrypt_at_rest is false
|
||||
unenc_state = {"config": {"encrypt_at_rest": False, "public_key": pub}}
|
||||
res_plain = StorageCryptoManager.encrypt_field_for_storage(plain_msg, server_state=unenc_state, is_unlocked=False)
|
||||
self.assertEqual(res_plain, plain_msg)
|
||||
|
||||
def test_storage_engine_sqlite_init_and_telemetry(self):
|
||||
"""Tests SQLite database initialization, WAL settings, column migrations, and telemetry."""
|
||||
db = StorageEngine.init_db(self.db_path, enrollment_secret="sec-test-123", max_seats=50)
|
||||
self.assertEqual(db, self.db_path)
|
||||
self.assertTrue(os.path.exists(db))
|
||||
|
||||
# Re-running init_db executes migrations smoothly
|
||||
db_reinit = StorageEngine.init_db(self.db_path, enrollment_secret="sec-test-123", max_seats=50)
|
||||
self.assertEqual(db_reinit, self.db_path)
|
||||
|
||||
# Check telemetry info
|
||||
state = {"config": {"db_path": self.db_path, "db_type": "sqlite"}}
|
||||
info = StorageEngine.get_storage_engine_info(self.db_path, server_state=state, is_unlocked=True)
|
||||
self.assertEqual(info["engine"], "sqlite")
|
||||
self.assertFalse(info["is_enterprise"])
|
||||
self.assertIn("active_issues", info["table_stats"])
|
||||
self.assertIn("clients", info["table_stats"])
|
||||
|
||||
def test_storage_engine_postgres_mocked(self):
|
||||
"""Tests Postgres schema initialization and SQLite-to-Postgres data migration with mocked drivers."""
|
||||
pg_cfg = {
|
||||
"host": "127.0.0.1",
|
||||
"port": 5432,
|
||||
"database": "logar_test",
|
||||
"user": "logar_user",
|
||||
"password": "secret_password",
|
||||
"sslmode": "disable"
|
||||
}
|
||||
|
||||
# Initialize SQLite DB with test records
|
||||
StorageEngine.init_db(self.db_path, enrollment_secret="sec-migrate", max_seats=10)
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
conn.execute("""
|
||||
INSERT INTO active_issues (fingerprint, site_name, server, signature, severity, message, os_type, first_seen, last_seen, run_count, status, last_run_id)
|
||||
VALUES ('fp1', 'SiteA', 'srv1', 'Sig1', 'HIGH', 'Test warning message', 'linux', '2026-01-01', '2026-01-02', 1, 'active', 'run1')
|
||||
""")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
# Mock psycopg2
|
||||
mock_psycopg2 = MagicMock()
|
||||
mock_conn = MagicMock()
|
||||
mock_cur = MagicMock()
|
||||
mock_conn.cursor.return_value.__enter__.return_value = mock_cur
|
||||
mock_psycopg2.connect.return_value = mock_conn
|
||||
|
||||
with patch.dict(sys.modules, {"psycopg2": mock_psycopg2}):
|
||||
# Test Postgres DB init
|
||||
ok, msg = StorageEngine.init_postgres_db(pg_cfg)
|
||||
self.assertTrue(ok)
|
||||
self.assertIn("successfully initialized", msg)
|
||||
|
||||
# Test Migration
|
||||
encrypt_mock = lambda x: f"ENC({x})"
|
||||
ok, msg, counts = StorageEngine.migrate_sqlite_to_postgres(self.db_path, pg_cfg, encrypt_fn=encrypt_mock)
|
||||
self.assertTrue(ok)
|
||||
self.assertIn("active_issues", counts)
|
||||
self.assertEqual(counts["active_issues"], 1)
|
||||
|
||||
# Test Postgres info retrieval
|
||||
mock_cur.fetchone.return_value = (10,)
|
||||
state = {"config": {"db_type": "postgres", "postgres": pg_cfg}}
|
||||
info = StorageEngine.get_storage_engine_info(None, server_state=state, is_unlocked=True)
|
||||
self.assertEqual(info["engine"], "postgres")
|
||||
self.assertTrue(info["is_enterprise"])
|
||||
self.assertEqual(info["table_stats"]["active_issues"], 10)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,173 @@
|
||||
"""
|
||||
Comprehensive unit tests for LOGAR UpdateManager, AuthManager, and IngestQueue.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from src.hub.update_manager import ServerUpdateManager
|
||||
from src.hub.auth_manager import PasswordManager, TOTPManager, SessionManager, RateLimiter
|
||||
from src.hub.ingest_queue import IngestQueueManager
|
||||
|
||||
|
||||
class TestHubUpdateAuthDeep(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.update_dir = os.path.join(self.temp_dir, "updates")
|
||||
self.db_path = os.path.join(self.temp_dir, "auth_test.db")
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.temp_dir, ignore_errors=True)
|
||||
|
||||
def test_update_manager_caching_and_manifest(self):
|
||||
"""Tests package registration, SHA256 integrity, local caching, and download serving."""
|
||||
manager = ServerUpdateManager(update_dir=self.update_dir, check_enabled=False)
|
||||
|
||||
# Create dummy binaries
|
||||
win_bin = os.path.join(self.temp_dir, "Win_Client.exe")
|
||||
linux_bin = os.path.join(self.temp_dir, "Linux_Client")
|
||||
with open(win_bin, "wb") as f:
|
||||
f.write(b"MZ_TEST_WIN_CLIENT_BINARY_2026")
|
||||
with open(linux_bin, "wb") as f:
|
||||
f.write(b"\x7fELF_TEST_LINUX_CLIENT_BINARY_2026")
|
||||
|
||||
# Register packages
|
||||
res_win = manager.register_client_package(
|
||||
os_type="windows",
|
||||
file_path=win_bin,
|
||||
version="2.5.0",
|
||||
filename="Win_Client.exe"
|
||||
)
|
||||
self.assertEqual(res_win.get("version"), "2.5.0")
|
||||
self.assertEqual(res_win.get("os"), "windows")
|
||||
self.assertIsNotNone(res_win.get("sha256"))
|
||||
|
||||
res_linux = manager.register_client_package(
|
||||
os_type="linux",
|
||||
file_path=linux_bin,
|
||||
version="2.5.0",
|
||||
filename="Linux_Client"
|
||||
)
|
||||
self.assertEqual(res_linux.get("os"), "linux")
|
||||
|
||||
# Get hosted packages
|
||||
packages = manager.get_hosted_packages()
|
||||
self.assertIn("windows", packages)
|
||||
self.assertIn("linux", packages)
|
||||
self.assertEqual(packages["windows"]["version"], "2.5.0")
|
||||
|
||||
# Test download path resolution
|
||||
pkg_res = manager.get_client_package_path(os_type="windows")
|
||||
self.assertIsNotNone(pkg_res)
|
||||
win_path, win_filename, win_sha = pkg_res
|
||||
self.assertTrue(os.path.exists(win_path))
|
||||
self.assertEqual(win_filename, "Win_Client.exe")
|
||||
|
||||
# Test update decision logic
|
||||
decision = manager.check_client_update(client_id="node-win-01", current_version="1.0.0", os_type="windows")
|
||||
self.assertTrue(decision.get("update_available"))
|
||||
self.assertEqual(decision.get("latest_version"), "2.5.0")
|
||||
|
||||
# Test download and caching from simulated upstream assets
|
||||
mock_assets = [
|
||||
{"name": "LOGAR_Client_Setup.exe", "download_url": "http://mock.upstream/LOGAR_Client_Setup.exe"},
|
||||
{"name": "Linux_Client.bin", "download_url": "http://mock.upstream/Linux_Client.bin"}
|
||||
]
|
||||
class MockHttpResp:
|
||||
status = 200
|
||||
def __init__(self):
|
||||
self._read_done = False
|
||||
def __enter__(self):
|
||||
return self
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
def read(self, size=65536):
|
||||
if not self._read_done:
|
||||
self._read_done = True
|
||||
return b"MOCK_BINARY_DATA_PAYLOAD_2026"
|
||||
return b""
|
||||
|
||||
with patch("urllib.request.urlopen", side_effect=lambda req, timeout=60: MockHttpResp()):
|
||||
cache_res = manager.download_and_cache_client_packages(assets=mock_assets, target_version="2.6.0")
|
||||
self.assertTrue(cache_res["windows"])
|
||||
self.assertTrue(cache_res["linux"])
|
||||
self.assertEqual(len(cache_res["downloaded"]), 2)
|
||||
self.assertEqual(manager.hosted_client_version, "2.6.0")
|
||||
|
||||
def test_auth_manager_components(self):
|
||||
"""Tests PasswordManager, TOTPManager, SessionManager, and RateLimiter."""
|
||||
# 1. PasswordManager
|
||||
pwd = "Enterprise_Admin_Password_!2026"
|
||||
pwd_hash, salt = PasswordManager.hash_password(pwd)
|
||||
self.assertTrue(PasswordManager.verify_password(pwd, salt, pwd_hash))
|
||||
self.assertFalse(PasswordManager.verify_password("WrongPassword", salt, pwd_hash))
|
||||
|
||||
# 2. TOTPManager
|
||||
secret = TOTPManager.generate_totp_secret()
|
||||
self.assertTrue(len(secret) >= 16)
|
||||
uri = TOTPManager.get_totp_uri("admin", secret, "LOGAR Fleet Hub")
|
||||
self.assertIn("otpauth://totp/", uri)
|
||||
|
||||
token = TOTPManager.get_totp_token(secret)
|
||||
self.assertEqual(len(token), 6)
|
||||
self.assertTrue(TOTPManager.verify_totp_code(secret, token))
|
||||
self.assertFalse(TOTPManager.verify_totp_code(secret, "000000" if token != "000000" else "111111"))
|
||||
|
||||
# 3. SessionManager
|
||||
session_token = SessionManager.generate_session_token("admin", db_path=self.db_path)
|
||||
self.assertIsNotNone(session_token)
|
||||
username = SessionManager.verify_session_token(session_token, db_path=self.db_path)
|
||||
self.assertEqual(username, "admin")
|
||||
self.assertIsNone(SessionManager.verify_session_token("invalid:token:format", db_path=self.db_path))
|
||||
|
||||
# 4. RateLimiter
|
||||
limiter = RateLimiter()
|
||||
ip = "192.168.1.100"
|
||||
for _ in range(5):
|
||||
self.assertTrue(limiter.check_ip_rate_limit(ip, max_attempts=5, window_seconds=60, db_path=self.db_path))
|
||||
limiter.record_failed_ip_attempt(ip, db_path=self.db_path)
|
||||
# 6th attempt exceeded
|
||||
self.assertFalse(limiter.check_ip_rate_limit(ip, max_attempts=5, window_seconds=60, db_path=self.db_path))
|
||||
|
||||
# Reset
|
||||
limiter.reset_failed_ip_attempts(ip, db_path=self.db_path)
|
||||
self.assertTrue(limiter.check_ip_rate_limit(ip, max_attempts=5, window_seconds=60, db_path=self.db_path))
|
||||
|
||||
def test_ingest_queue_lifecycle_and_batching(self):
|
||||
"""Tests IngestQueueManager push, stats, and flush callback."""
|
||||
processed_items = []
|
||||
|
||||
def mock_processor(payload, db_path, window_hours, min_runs):
|
||||
processed_items.extend(payload.get("logs", []))
|
||||
return {"processed": len(payload.get("logs", [])), "promoted_verified": 0}
|
||||
|
||||
queue = IngestQueueManager(flush_interval=0.05, batch_size=10, max_queue=100)
|
||||
|
||||
# Enqueue items
|
||||
for i in range(15):
|
||||
pushed = queue.push({"server": "node1", "logs": [{"id": i, "data": f"log_{i}"}]})
|
||||
self.assertTrue(pushed)
|
||||
|
||||
stats = queue.get_stats()
|
||||
self.assertEqual(stats["queue_size"], 15)
|
||||
self.assertEqual(stats["total_ingested"], 15)
|
||||
|
||||
# Flush
|
||||
flush_res = queue.flush(db_path=self.db_path, process_callback=mock_processor)
|
||||
self.assertEqual(flush_res["status"], "success")
|
||||
self.assertEqual(flush_res["flushed_runs"], 15)
|
||||
self.assertEqual(flush_res["processed_logs"], 15)
|
||||
self.assertEqual(len(processed_items), 15)
|
||||
|
||||
stats_after = queue.get_stats()
|
||||
self.assertEqual(stats_after["queue_size"], 0)
|
||||
self.assertEqual(stats_after["total_flushed"], 15)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -205,6 +205,94 @@ class TestServerLicensingAndWebUI(unittest.TestCase):
|
||||
self.assertEqual(info["seats"], 20)
|
||||
self.assertEqual(info["license_id"], "LIC-ACT-20")
|
||||
|
||||
def test_signature_verification_and_anti_forgery(self):
|
||||
"""Verifies cryptographic signature validation against master root CA and detection of forged certificates."""
|
||||
csr_pem, _ = enrollment.generate_ca_csr(cert_dir=self.server_cert_dir)
|
||||
sub_pem, root_pem = _create_mock_signed_license_cert(csr_pem, seats=30, license_id="LIC-VALID-30")
|
||||
|
||||
# 1. Valid signature against genuine root CA
|
||||
self.assertTrue(enrollment.verify_certificate_signature(sub_pem, root_pem))
|
||||
info_valid = enrollment.extract_license_info_from_cert(sub_pem, upstream_ca_pem=root_pem)
|
||||
self.assertTrue(info_valid["signature_verified"])
|
||||
self.assertEqual(info_valid["seats"], 30)
|
||||
|
||||
# 2. Fake / Rogue Root CA trying to verify the same cert
|
||||
fake_csr, _ = enrollment.generate_ca_csr(cert_dir=self.server_cert_dir)
|
||||
_, fake_root_pem = _create_mock_signed_license_cert(fake_csr, seats=999, license_id="LIC-FAKE")
|
||||
self.assertFalse(enrollment.verify_certificate_signature(sub_pem, fake_root_pem))
|
||||
info_invalid = enrollment.extract_license_info_from_cert(sub_pem, upstream_ca_pem=fake_root_pem)
|
||||
self.assertFalse(info_invalid["signature_verified"])
|
||||
|
||||
def test_database_tampering_seat_reconciliation(self):
|
||||
"""Verifies that direct DB manipulation of max_seats is reconciled against the certified X.509 quota."""
|
||||
from license_manager import LicenseManager
|
||||
|
||||
csr_pem, _ = enrollment.generate_ca_csr(cert_dir=self.server_cert_dir)
|
||||
sub_pem, root_pem = _create_mock_signed_license_cert(csr_pem, seats=25, license_id="LIC-SEAT-25")
|
||||
|
||||
# Save certified ca.crt and upstream_ca.crt
|
||||
ca_path = os.path.join(self.server_cert_dir, "ca.crt")
|
||||
with open(ca_path, "w", encoding="utf-8") as f:
|
||||
f.write(sub_pem)
|
||||
with open(os.path.join(self.server_cert_dir, "upstream_ca.crt"), "w", encoding="utf-8") as f:
|
||||
f.write(root_pem)
|
||||
|
||||
Server.init_db(self.server_db, max_seats=25)
|
||||
server_state = {
|
||||
"config": {
|
||||
"db_path": self.server_db,
|
||||
"cert_dir": self.server_cert_dir,
|
||||
"max_seats": 25
|
||||
}
|
||||
}
|
||||
|
||||
# Attacker attempts direct SQLite manipulation: SET max_seats = 99999
|
||||
conn = sqlite3.connect(self.server_db)
|
||||
conn.execute("UPDATE license_config SET max_seats = 99999 WHERE id = 1")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
# LicenseManager inspects state and auto-reconciles max_seats to certified 25
|
||||
summary = LicenseManager.get_license_summary(server_state=server_state, db_path=self.server_db)
|
||||
self.assertEqual(summary["max_seats"], 25)
|
||||
self.assertTrue(summary["signature_verified"])
|
||||
|
||||
# Check DB was reconciled
|
||||
conn = sqlite3.connect(self.server_db)
|
||||
row = conn.execute("SELECT max_seats FROM license_config WHERE id = 1").fetchone()
|
||||
conn.close()
|
||||
self.assertEqual(row[0], 25)
|
||||
|
||||
def test_import_license_signature_validation(self):
|
||||
"""Verifies that /api/license/import rejects certificates not signed by the upstream authority."""
|
||||
from fastapi import HTTPException
|
||||
csr_pem, _ = enrollment.generate_ca_csr(cert_dir=self.server_cert_dir)
|
||||
sub_pem, root_pem = _create_mock_signed_license_cert(csr_pem, seats=50, license_id="LIC-MOCK-50")
|
||||
|
||||
# Generate mismatched root CA
|
||||
fake_csr, _ = enrollment.generate_ca_csr(cert_dir=self.server_cert_dir)
|
||||
_, rogue_root_pem = _create_mock_signed_license_cert(fake_csr, seats=10, license_id="LIC-ROGUE")
|
||||
|
||||
Server.init_db(self.server_db, max_seats=5)
|
||||
Server.SERVER_STATE["config"] = {
|
||||
"db_path": self.server_db,
|
||||
"cert_dir": self.server_cert_dir,
|
||||
"max_seats": 5
|
||||
}
|
||||
|
||||
# Attempt to import sub_pem with mismatched upstream_ca -> must raise HTTPException 400
|
||||
req_bad = Server.LicenseImportRequest(ca_cert=sub_pem, upstream_ca=rogue_root_pem)
|
||||
with self.assertRaises(HTTPException) as ctx:
|
||||
Server.import_signed_license(req_bad, auth_user="admin")
|
||||
self.assertEqual(ctx.exception.status_code, 400)
|
||||
self.assertIn("Cryptographic verification failed", ctx.exception.detail)
|
||||
|
||||
# Import sub_pem with valid matching upstream_ca -> must succeed
|
||||
req_good = Server.LicenseImportRequest(ca_cert=sub_pem, upstream_ca=root_pem)
|
||||
res = Server.import_signed_license(req_good, auth_user="admin")
|
||||
self.assertEqual(res["status"], "success")
|
||||
self.assertEqual(res["seats"], 50)
|
||||
|
||||
def test_webui_rendering(self):
|
||||
"""Verifies that the WebUI HTML contains key components, stats, and download triggers."""
|
||||
Server.init_db(self.server_db, max_seats=10)
|
||||
@@ -219,15 +307,58 @@ class TestServerLicensingAndWebUI(unittest.TestCase):
|
||||
"license_server_url": "https://logar.eibl.tech"
|
||||
}
|
||||
|
||||
response = Server.serve_webui()
|
||||
html = response.body.decode("utf-8")
|
||||
self.assertIn("LOGAR Fleet Hub", html)
|
||||
self.assertNotIn("LOGAR Fleet Hub & License Management", html)
|
||||
self.assertNotIn("Export .lic Request", html)
|
||||
self.assertNotIn("Import License (.lic / ca.crt)", html)
|
||||
self.assertIn("Sync & Renew via mTLS", html)
|
||||
self.assertIn("logar.eibl.tech", html)
|
||||
def test_auto_update_lock_and_package_upload_protection(self):
|
||||
"""Verifies that unlicensed servers block package upload with 403 and render locked UI."""
|
||||
from fastapi import HTTPException
|
||||
Server.init_db(self.server_db, max_seats=0)
|
||||
Server.SERVER_STATE["config"] = {
|
||||
"db_path": self.server_db,
|
||||
"cert_dir": self.server_cert_dir,
|
||||
"max_seats": 0
|
||||
}
|
||||
|
||||
# Ensure license_config has no auto_update
|
||||
conn = sqlite3.connect(self.server_db)
|
||||
conn.execute("UPDATE license_config SET license_status = 'unlicensed', features = '[]' WHERE id = 1")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
# WebUI should render locked banner and disabled upload controls
|
||||
resp = Server.serve_webui()
|
||||
html = resp.body.decode("utf-8")
|
||||
self.assertIn("Fleet Auto-Update is Inactive & Locked", html)
|
||||
self.assertIn("disabled", html)
|
||||
|
||||
def test_json_lic_package_import(self):
|
||||
"""Verifies importing an activated .lic JSON package via /api/license/import."""
|
||||
csr_pem, _ = enrollment.generate_ca_csr(cert_dir=self.server_cert_dir)
|
||||
sub_pem, root_pem = _create_mock_signed_license_cert(
|
||||
csr_pem, seats=30, license_id="LIC-TEST-JSON-01"
|
||||
)
|
||||
|
||||
Server.init_db(self.server_db, max_seats=5)
|
||||
Server.SERVER_STATE["config"] = {
|
||||
"db_path": self.server_db,
|
||||
"cert_dir": self.server_cert_dir,
|
||||
"max_seats": 5
|
||||
}
|
||||
|
||||
# JSON package format
|
||||
lic_pkg = {
|
||||
"format": "LOGAR_LICENSE_PACKAGE",
|
||||
"license_id": "LIC-TEST-JSON-01",
|
||||
"seats": 30,
|
||||
"features": ["enterprise_db", "auto_update"],
|
||||
"ca_cert": sub_pem,
|
||||
"master_root_ca": root_pem
|
||||
}
|
||||
|
||||
req = Server.LicenseImportRequest(lic=json.dumps(lic_pkg))
|
||||
res = Server.import_signed_license(req, auth_user="admin")
|
||||
self.assertEqual(res["status"], "success")
|
||||
self.assertEqual(res["seats"], 30)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
@@ -69,7 +69,7 @@ def run_tests():
|
||||
|
||||
# 1. Check current license status
|
||||
req_status_init = urllib.request.Request(f"http://{HERMES_HOST}:{HERMES_PORT}/api/license/status")
|
||||
with urllib.request.urlopen(req_status_init, timeout=5) as resp:
|
||||
with urllib.request.urlopen(req_status_init, timeout=15) as resp:
|
||||
init_lic_stat = json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
if init_lic_stat.get("status") == "UNLICENSED":
|
||||
@@ -85,7 +85,7 @@ def run_tests():
|
||||
headers={"Content-Type": "application/json"}
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req_unlicensed, timeout=5):
|
||||
with urllib.request.urlopen(req_unlicensed, timeout=15):
|
||||
assert False, "Expected HTTP 403 on unlicensed server enrollment"
|
||||
except urllib.error.HTTPError as e:
|
||||
assert e.code == 403, f"Expected HTTP 403 on unlicensed server, got {e.code}"
|
||||
@@ -96,7 +96,7 @@ def run_tests():
|
||||
f"http://{HERMES_HOST}:{HERMES_PORT}/api/license/csr",
|
||||
headers={"Authorization": f"Bearer {auth_token}"}
|
||||
)
|
||||
with urllib.request.urlopen(req_csr, timeout=5) as resp:
|
||||
with urllib.request.urlopen(req_csr, timeout=15) as resp:
|
||||
assert resp.status == 200
|
||||
csr_data = json.loads(resp.read().decode("utf-8"))
|
||||
csr_pem = csr_data["csr"]
|
||||
@@ -121,7 +121,7 @@ def run_tests():
|
||||
data=json.dumps(import_payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {auth_token}"}
|
||||
)
|
||||
with urllib.request.urlopen(req_import, timeout=5) as resp:
|
||||
with urllib.request.urlopen(req_import, timeout=15) as resp:
|
||||
assert resp.status == 200
|
||||
import_res = json.loads(resp.read().decode("utf-8"))
|
||||
assert import_res["status"] == "success"
|
||||
@@ -130,7 +130,7 @@ def run_tests():
|
||||
|
||||
# 5. Verify license status
|
||||
req_status = urllib.request.Request(f"http://{HERMES_HOST}:{HERMES_PORT}/api/license/status")
|
||||
with urllib.request.urlopen(req_status, timeout=5) as resp:
|
||||
with urllib.request.urlopen(req_status, timeout=15) as resp:
|
||||
assert resp.status == 200
|
||||
lic_stat = json.loads(resp.read().decode("utf-8"))
|
||||
assert lic_stat["status"] == "ACTIVE"
|
||||
@@ -153,7 +153,7 @@ def run_tests():
|
||||
headers={"Content-Type": "application/json"}
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req_bad, timeout=5):
|
||||
with urllib.request.urlopen(req_bad, timeout=15):
|
||||
assert False, "Expected HTTP 403 on invalid secret"
|
||||
except urllib.error.HTTPError as e:
|
||||
assert e.code == 403, f"Expected HTTP 403, got {e.code}"
|
||||
@@ -171,7 +171,7 @@ def run_tests():
|
||||
data=json.dumps(valid_enroll_payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"}
|
||||
)
|
||||
with urllib.request.urlopen(req_valid, timeout=5) as resp:
|
||||
with urllib.request.urlopen(req_valid, timeout=15) as resp:
|
||||
assert resp.status == 200, f"Expected 200, got {resp.status}"
|
||||
enroll_data = json.loads(resp.read().decode("utf-8"))
|
||||
assert "ca_cert" in enroll_data
|
||||
@@ -194,7 +194,7 @@ def run_tests():
|
||||
auth_tok = server_conf.get("auth_token") or client_conf.get("auth_token")
|
||||
client_headers = {"Authorization": f"Bearer {auth_tok}"} if auth_tok else {}
|
||||
clients_req = urllib.request.Request(f"http://{HERMES_HOST}:{HERMES_PORT}/api/clients", headers=client_headers)
|
||||
with urllib.request.urlopen(clients_req, timeout=5) as resp:
|
||||
with urllib.request.urlopen(clients_req, timeout=15) as resp:
|
||||
clients_data = json.loads(resp.read().decode("utf-8"))
|
||||
assert clients_data["active_seats"] >= 1
|
||||
found_c = any(c["client_id"] == client_id for c in clients_data["clients"])
|
||||
@@ -294,7 +294,7 @@ def run_tests():
|
||||
token = server_conf.get("auth_token") or client_conf.get("auth_token")
|
||||
headers = {"Authorization": f"Bearer {token}"} if token else {}
|
||||
req = urllib.request.Request(f"http://{HERMES_HOST}:{HERMES_PORT}/api/hermes/report", headers=headers)
|
||||
with urllib.request.urlopen(req, timeout=5) as response:
|
||||
with urllib.request.urlopen(req, timeout=15) as response:
|
||||
assert response.status == 200, f"Expected 200, got {response.status}"
|
||||
hermes_data = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
|
||||
@@ -510,8 +510,485 @@ class TestServerComponent(unittest.TestCase):
|
||||
info_disabled = Server.get_user_totp_info("admin", db_path=self.test_db)
|
||||
self.assertFalse(info_disabled["enabled"])
|
||||
|
||||
def test_persistent_session_secret_and_rate_limiting(self):
|
||||
"""Tests DB persistence of session secret and sliding window rate limiting."""
|
||||
Server.init_db(self.test_db)
|
||||
|
||||
# 1. Session secret persistence
|
||||
sec1 = Server.get_session_secret(self.test_db)
|
||||
self.assertTrue(len(sec1) >= 32)
|
||||
conn = sqlite3.connect(self.test_db)
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT value FROM server_meta WHERE key = 'session_secret'")
|
||||
row = c.fetchone()
|
||||
conn.close()
|
||||
self.assertIsNotNone(row)
|
||||
self.assertEqual(row[0], sec1)
|
||||
|
||||
# Token generated with test_db
|
||||
token = Server.generate_session_token("admin", db_path=self.test_db)
|
||||
self.assertEqual(Server.verify_session_token(token, db_path=self.test_db), "admin")
|
||||
|
||||
# 2. Rate limiting
|
||||
test_ip = "10.0.0.99"
|
||||
Server.reset_failed_ip_attempts(test_ip, scope="login", db_path=self.test_db)
|
||||
|
||||
self.assertTrue(Server.check_ip_rate_limit(test_ip, max_attempts=3, window_seconds=60, scope="login", db_path=self.test_db))
|
||||
Server.record_failed_ip_attempt(test_ip, scope="login", db_path=self.test_db)
|
||||
Server.record_failed_ip_attempt(test_ip, scope="login", db_path=self.test_db)
|
||||
self.assertTrue(Server.check_ip_rate_limit(test_ip, max_attempts=3, window_seconds=60, scope="login", db_path=self.test_db))
|
||||
|
||||
Server.record_failed_ip_attempt(test_ip, scope="login", db_path=self.test_db)
|
||||
self.assertFalse(Server.check_ip_rate_limit(test_ip, max_attempts=3, window_seconds=60, scope="login", db_path=self.test_db))
|
||||
|
||||
# Different scope ('enroll') should still be allowed
|
||||
self.assertTrue(Server.check_ip_rate_limit(test_ip, max_attempts=3, window_seconds=60, scope="enroll", db_path=self.test_db))
|
||||
|
||||
Server.reset_failed_ip_attempts(test_ip, scope="login", db_path=self.test_db)
|
||||
self.assertTrue(Server.check_ip_rate_limit(test_ip, max_attempts=3, window_seconds=60, scope="login", db_path=self.test_db))
|
||||
|
||||
def test_ingest_queue_manager_and_batch_flush(self):
|
||||
"""Tests IngestQueueManager buffering, capacity bounds, and atomic batch flushing."""
|
||||
Server.init_db(self.test_db)
|
||||
qm = Server.IngestQueueManager(flush_interval=1.0, batch_size=5, max_queue=50)
|
||||
|
||||
# Initial stats
|
||||
stats = qm.get_stats()
|
||||
self.assertEqual(stats["queue_size"], 0)
|
||||
self.assertEqual(stats["total_ingested"], 0)
|
||||
self.assertEqual(stats["total_flushed"], 0)
|
||||
|
||||
# Push 3 payloads
|
||||
for i in range(3):
|
||||
payload = {
|
||||
"server": f"node-{i}.corp.local",
|
||||
"logs": [
|
||||
{"server": f"node-{i}", "signature": f"Sig_{i}", "severity": "WARNING", "message": f"Log msg {i}", "os_type": "linux"}
|
||||
]
|
||||
}
|
||||
pushed = qm.push(payload)
|
||||
self.assertTrue(pushed)
|
||||
|
||||
stats = qm.get_stats()
|
||||
self.assertEqual(stats["queue_size"], 3)
|
||||
self.assertEqual(stats["total_ingested"], 3)
|
||||
|
||||
# Flush to database
|
||||
res_flush = qm.flush(db_path=self.test_db, window_hours=12, min_runs=1)
|
||||
self.assertEqual(res_flush["flushed_runs"], 3)
|
||||
self.assertEqual(res_flush["processed_logs"], 3)
|
||||
|
||||
# Verify DB records
|
||||
conn = sqlite3.connect(self.test_db)
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT COUNT(*) FROM ingest_runs")
|
||||
run_count = c.fetchone()[0]
|
||||
c.execute("SELECT COUNT(*) FROM active_issues WHERE status = 'VERIFIED'")
|
||||
issue_count = c.fetchone()[0]
|
||||
conn.close()
|
||||
|
||||
self.assertEqual(run_count, 3)
|
||||
self.assertEqual(issue_count, 3)
|
||||
|
||||
# Stats after flush
|
||||
stats = qm.get_stats()
|
||||
self.assertEqual(stats["queue_size"], 0)
|
||||
self.assertEqual(stats["total_flushed"], 3)
|
||||
|
||||
def test_enterprise_db_feature_gating(self):
|
||||
"""Tests feature check helpers and DB storage info under standard and enterprise licensed states."""
|
||||
Server.init_db(self.test_db)
|
||||
|
||||
# 1. Standard / Unlicensed state
|
||||
self.assertFalse(Server.is_feature_licensed("enterprise_db", db_path=self.test_db))
|
||||
info = Server.get_storage_engine_info(db_path=self.test_db)
|
||||
self.assertEqual(info["engine"], "sqlite")
|
||||
self.assertFalse(info["feature_unlocked"])
|
||||
self.assertIn("SQLite 3 (WAL Mode)", info["backend_display"])
|
||||
|
||||
# 2. Unlock enterprise_db feature in DB
|
||||
conn = sqlite3.connect(self.test_db)
|
||||
conn.execute("UPDATE license_config SET features = ? WHERE id = 1", (json.dumps(["enterprise_db", "ai_heuristics"]),))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
self.assertTrue(Server.is_feature_licensed("enterprise_db", db_path=self.test_db))
|
||||
self.assertTrue(Server.is_feature_licensed("ai_heuristics", db_path=self.test_db))
|
||||
self.assertFalse(Server.is_feature_licensed("unknown_feat", db_path=self.test_db))
|
||||
|
||||
info_unlocked = Server.get_storage_engine_info(db_path=self.test_db)
|
||||
self.assertTrue(info_unlocked["feature_unlocked"])
|
||||
self.assertIn("enterprise_db", info_unlocked["licensed_features"])
|
||||
|
||||
def test_postgres_storage_endpoints_authorization(self):
|
||||
"""Tests REST API security gating on DB engine management endpoints."""
|
||||
Server.init_db(self.test_db)
|
||||
Server.SERVER_STATE["config"] = {"db_path": self.test_db, "storage_engine": "sqlite"}
|
||||
|
||||
# 1. Test direct authorization & endpoint logic
|
||||
stats = Server.api_db_stats()
|
||||
self.assertIn("queue", stats)
|
||||
self.assertIn("storage", stats)
|
||||
self.assertFalse(stats["enterprise_db_licensed"])
|
||||
|
||||
# 2. Test Connection without enterprise_db -> HTTPException 403
|
||||
pg_req = Server.DbConnectionTestRequest(host="localhost", port=5432, database="logar", user="postgres", password="")
|
||||
with self.assertRaises(Server.HTTPException) as cm:
|
||||
Server.api_db_test_connection(pg_req)
|
||||
self.assertEqual(cm.exception.status_code, 403)
|
||||
self.assertIn("enterprise_db", cm.exception.detail)
|
||||
|
||||
# 3. Switch backend without enterprise_db -> HTTPException 403
|
||||
switch_req = Server.DbSwitchBackendRequest(db_type="postgres", host="localhost", port=5432, database="logar", user="postgres", password="")
|
||||
with self.assertRaises(Server.HTTPException) as cm:
|
||||
Server.api_db_switch_backend(switch_req)
|
||||
self.assertEqual(cm.exception.status_code, 403)
|
||||
self.assertIn("enterprise_db", cm.exception.detail)
|
||||
|
||||
# 4. Flush buffer endpoint executes successfully
|
||||
flush_res = Server.api_db_flush_buffer()
|
||||
self.assertEqual(flush_res["status"], "success")
|
||||
|
||||
# 5. Optional HTTP client verification if starlette testclient dependencies are present
|
||||
try:
|
||||
from fastapi.testclient import TestClient
|
||||
client = TestClient(Server.app)
|
||||
|
||||
# Unauthenticated request to /api/db/stats -> 401
|
||||
res = client.get("/api/db/stats")
|
||||
self.assertEqual(res.status_code, 401)
|
||||
|
||||
# Authenticated request with session cookie
|
||||
token = Server.generate_session_token("admin", db_path=self.test_db)
|
||||
client.cookies.set(Server.SESSION_COOKIE_NAME, token)
|
||||
|
||||
res_auth = client.get("/api/db/stats")
|
||||
self.assertEqual(res_auth.status_code, 200)
|
||||
data = res_auth.json()
|
||||
self.assertIn("queue", data)
|
||||
self.assertIn("storage", data)
|
||||
self.assertFalse(data["enterprise_db_licensed"])
|
||||
|
||||
# 403 checks via HTTP client
|
||||
res_test = client.post("/api/db/test-connection", json={"host": "localhost", "port": 5432, "database": "logar", "user": "postgres", "password": ""})
|
||||
self.assertEqual(res_test.status_code, 403)
|
||||
|
||||
res_switch = client.post("/api/db/switch-backend", json={"db_type": "postgres", "host": "localhost", "port": 5432, "database": "logar", "user": "postgres", "password": ""})
|
||||
self.assertEqual(res_switch.status_code, 403)
|
||||
|
||||
res_flush = client.post("/api/db/flush-buffer")
|
||||
self.assertEqual(res_flush.status_code, 200)
|
||||
except (ImportError, RuntimeError, Exception):
|
||||
# Starlette TestClient may raise RuntimeError when httpx/httpx2 is missing in lightweight runner environments
|
||||
pass
|
||||
|
||||
def test_db_encryption_at_rest_and_full_config_api(self):
|
||||
"""Tests OpenPGP data-at-rest encryption/decryption and full DB/Cache configuration API."""
|
||||
Server.init_db(self.test_db)
|
||||
priv_armored, pub_armored, fp = Server.generate_server_keypair("test-db-node")
|
||||
priv_key, _ = pgpy.PGPKey.from_blob(priv_armored)
|
||||
pub_key, _ = pgpy.PGPKey.from_blob(pub_armored)
|
||||
|
||||
Server.SERVER_STATE["config"] = {
|
||||
"db_path": self.test_db,
|
||||
"storage_engine": "sqlite",
|
||||
"cache_mode": "direct",
|
||||
"encrypt_at_rest": False,
|
||||
"private_key": priv_armored,
|
||||
"public_key": pub_armored,
|
||||
"server_fingerprint": fp
|
||||
}
|
||||
Server.SERVER_STATE["private_key_obj"] = priv_key
|
||||
Server.SERVER_STATE["public_key_obj"] = pub_key
|
||||
|
||||
# 1. Test OpenPGP encryption & decryption roundtrip
|
||||
raw_msg = "Critical database corruption: segment 0x89FA damaged"
|
||||
encrypted_msg = Server.encrypt_field_for_storage(raw_msg, force=True)
|
||||
self.assertTrue(encrypted_msg.startswith("-----BEGIN PGP MESSAGE-----"))
|
||||
self.assertNotIn("Critical database corruption", encrypted_msg)
|
||||
|
||||
decrypted_msg = Server.decrypt_field_from_storage(encrypted_msg)
|
||||
self.assertEqual(decrypted_msg, raw_msg)
|
||||
|
||||
# 2. Test that without enterprise_db license, buffering, encryption at rest, and postgres are blocked
|
||||
cfg_unlicensed_buf = Server.DbFullConfigRequest(
|
||||
storage_engine="sqlite",
|
||||
cache_mode="buffered",
|
||||
encrypt_at_rest=False
|
||||
)
|
||||
with self.assertRaises(Server.HTTPException) as cm:
|
||||
Server.api_db_update_config(cfg_unlicensed_buf)
|
||||
self.assertEqual(cm.exception.status_code, 403)
|
||||
self.assertIn("enterprise_db", cm.exception.detail)
|
||||
|
||||
cfg_unlicensed_enc = Server.DbFullConfigRequest(
|
||||
storage_engine="sqlite",
|
||||
cache_mode="direct",
|
||||
encrypt_at_rest=True
|
||||
)
|
||||
with self.assertRaises(Server.HTTPException) as cm:
|
||||
Server.api_db_update_config(cfg_unlicensed_enc)
|
||||
self.assertEqual(cm.exception.status_code, 403)
|
||||
self.assertIn("enterprise_db", cm.exception.detail)
|
||||
|
||||
pg_req = Server.DbFullConfigRequest(
|
||||
storage_engine="postgres",
|
||||
cache_mode="direct",
|
||||
pg_config=Server.DbConnectionTestRequest(host="db.corp.internal", port=5432, database="logar", user="postgres", password="")
|
||||
)
|
||||
with self.assertRaises(Server.HTTPException) as cm:
|
||||
Server.api_db_update_config(pg_req)
|
||||
self.assertEqual(cm.exception.status_code, 403)
|
||||
self.assertIn("enterprise_db", cm.exception.detail)
|
||||
|
||||
# 3. Standard local SQLite configuration succeeds when unlicensed
|
||||
cfg_standard = Server.DbFullConfigRequest(
|
||||
storage_engine="sqlite",
|
||||
cache_mode="direct",
|
||||
encrypt_at_rest=False
|
||||
)
|
||||
res_std = Server.api_db_update_config(cfg_standard)
|
||||
self.assertEqual(res_std["status"], "success")
|
||||
|
||||
# 4. Unlock enterprise_db feature in license_config
|
||||
conn = sqlite3.connect(self.test_db)
|
||||
conn.execute("UPDATE license_config SET features = ? WHERE id = 1", (json.dumps(["enterprise_db"]),))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
# 5. Test updating config with enterprise_db unlocked (direct -> buffered, encrypt_at_rest = True)
|
||||
cfg_req = Server.DbFullConfigRequest(
|
||||
storage_engine="sqlite",
|
||||
cache_mode="buffered",
|
||||
flush_interval=4.0,
|
||||
batch_size=200,
|
||||
max_queue=25000,
|
||||
encrypt_at_rest=True
|
||||
)
|
||||
res = Server.api_db_update_config(cfg_req)
|
||||
self.assertEqual(res["status"], "success")
|
||||
self.assertEqual(Server.SERVER_STATE["config"]["cache_mode"], "buffered")
|
||||
self.assertTrue(Server.SERVER_STATE["config"]["encrypt_at_rest"])
|
||||
self.assertEqual(Server.INGEST_QUEUE.flush_interval, 4.0)
|
||||
self.assertEqual(Server.INGEST_QUEUE.batch_size, 200)
|
||||
|
||||
# 6. Verify storage engine info output
|
||||
info = Server.get_storage_engine_info(db_path=self.test_db)
|
||||
self.assertEqual(info["cache_mode"], "buffered")
|
||||
self.assertTrue(info["encrypt_at_rest"])
|
||||
self.assertEqual(info["flush_interval"], 4.0)
|
||||
|
||||
def test_active_licensed_features_helper(self):
|
||||
"""Tests helper function to extract active licensed capabilities from DB and certificates."""
|
||||
Server.init_db(self.test_db)
|
||||
Server.SERVER_STATE["ca_cert_pem"] = ""
|
||||
Server.SERVER_STATE["config"] = {"db_path": self.test_db, "cert_dir": "non_existent_test_certs"}
|
||||
|
||||
# 1. Unlicensed state
|
||||
features_unlicensed = Server.get_active_licensed_features(db_path=self.test_db)
|
||||
self.assertEqual(features_unlicensed, [])
|
||||
self.assertFalse(Server.is_feature_licensed("enterprise_db", db_path=self.test_db))
|
||||
|
||||
# 2. Add features via JSON string in DB
|
||||
conn = sqlite3.connect(self.test_db)
|
||||
conn.execute("UPDATE license_config SET features = ? WHERE id = 1", (json.dumps(["enterprise_db", "ai_heuristics"]),))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
features_licensed = Server.get_active_licensed_features(db_path=self.test_db)
|
||||
self.assertEqual(features_licensed, ["ai_heuristics", "enterprise_db"])
|
||||
self.assertTrue(Server.is_feature_licensed("enterprise_db", db_path=self.test_db))
|
||||
self.assertTrue(Server.is_feature_licensed("ai_heuristics", db_path=self.test_db))
|
||||
self.assertFalse(Server.is_feature_licensed("non_existent_feature", db_path=self.test_db))
|
||||
|
||||
# 3. Verify get_license_summary includes features list
|
||||
summary = Server.get_license_summary()
|
||||
self.assertIn("features", summary)
|
||||
self.assertIn("enterprise_db", summary["features"])
|
||||
|
||||
def test_webui_tab_rendering_and_feature_cards(self):
|
||||
"""Tests WebUI HTML generation for clean tab buttons and feature cards in PKI tab."""
|
||||
Server.init_db(self.test_db)
|
||||
Server.SERVER_STATE["config"] = {
|
||||
"db_path": self.test_db,
|
||||
"cert_dir": "certs",
|
||||
"tcp_port": 9443,
|
||||
"hermes_port": 8443,
|
||||
"server_name": "Test-Hub"
|
||||
}
|
||||
|
||||
# 1. Render WebUI in standard / unlicensed mode
|
||||
resp = Server.serve_webui(request=None)
|
||||
html_content = resp.body.decode("utf-8")
|
||||
|
||||
# Verify clean tab buttons without number suffixes
|
||||
self.assertIn('>💻 Fleet Forwarders</button>', html_content)
|
||||
self.assertIn('>🤖 Hermes AI & Tokens</button>', html_content)
|
||||
self.assertIn('>📊 Verified Telemetry</button>', html_content)
|
||||
self.assertIn('>📜 Sub-CA & Licensing</button>', html_content)
|
||||
self.assertIn('>💾 DB Engine</button>', html_content)
|
||||
self.assertNotIn('🟢 UNLOCKED', html_content)
|
||||
self.assertNotIn('Fleet Forwarders (', html_content)
|
||||
|
||||
# Verify Licensed Capabilities card is present
|
||||
self.assertIn('Licensed Capabilities & Feature Modules', html_content)
|
||||
self.assertIn('Standard Community Edition is running', html_content)
|
||||
|
||||
# 2. Render WebUI with enterprise_db unlocked
|
||||
conn = sqlite3.connect(self.test_db)
|
||||
conn.execute("UPDATE license_config SET features = ? WHERE id = 1", (json.dumps(["enterprise_db"]),))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
resp_ent = Server.serve_webui(request=None)
|
||||
html_ent = resp_ent.body.decode("utf-8")
|
||||
self.assertIn('enterprise_db', html_ent)
|
||||
self.assertIn('Enterprise Database & High-Throughput Engine', html_ent)
|
||||
self.assertIn('>💾 DB Engine</button>', html_ent)
|
||||
self.assertNotIn('🟢 UNLOCKED', html_ent)
|
||||
|
||||
def test_explicit_opt_in_guarantee(self):
|
||||
"""Tests that licensing does NOT automatically turn on encryption or buffering without explicit admin setting."""
|
||||
Server.init_db(self.test_db)
|
||||
priv_armored, pub_armored, fp = Server.generate_server_keypair("test-opt-in-node")
|
||||
priv_key, _ = pgpy.PGPKey.from_blob(priv_armored)
|
||||
pub_key, _ = pgpy.PGPKey.from_blob(pub_armored)
|
||||
|
||||
Server.SERVER_STATE["config"] = {
|
||||
"db_path": self.test_db,
|
||||
"cert_dir": "certs",
|
||||
"storage_engine": "sqlite",
|
||||
"cache_mode": "direct",
|
||||
"encrypt_at_rest": False,
|
||||
"private_key": priv_armored,
|
||||
"public_key": pub_armored,
|
||||
"server_fingerprint": fp
|
||||
}
|
||||
Server.SERVER_STATE["private_key_obj"] = priv_key
|
||||
Server.SERVER_STATE["public_key_obj"] = pub_key
|
||||
|
||||
# Unlock enterprise license
|
||||
conn = sqlite3.connect(self.test_db)
|
||||
conn.execute("UPDATE license_config SET features = ? WHERE id = 1", (json.dumps(["enterprise_db"]),))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
# License is active, but config has encrypt_at_rest=False -> plaintext storage
|
||||
raw_msg = "Sample operational log event"
|
||||
result_msg = Server.encrypt_field_for_storage(raw_msg)
|
||||
self.assertEqual(result_msg, raw_msg)
|
||||
self.assertFalse(result_msg.startswith("-----BEGIN PGP MESSAGE-----"))
|
||||
|
||||
# Ingestion cache mode is direct
|
||||
self.assertEqual(Server.SERVER_STATE["config"]["cache_mode"], "direct")
|
||||
|
||||
def test_reset_enrollment_secret_and_invalidation(self):
|
||||
"""Tests that rotating enrollment secret invalidates the previous secret and updates DB & config."""
|
||||
init_sec = "initial-secret-123456"
|
||||
Server.init_db(self.test_db, enrollment_secret=init_sec, max_seats=5)
|
||||
Server.SERVER_STATE["config"] = {
|
||||
"db_path": self.test_db,
|
||||
"enrollment_secret": init_sec,
|
||||
"config_file_path": self.test_config
|
||||
}
|
||||
with open(self.test_config, "w", encoding="utf-8") as f:
|
||||
json.dump({"enrollment_secret": init_sec, "db_path": self.test_db}, f)
|
||||
|
||||
# 1. Rotate enrollment secret
|
||||
new_sec = Server.rotate_enrollment_secret()
|
||||
self.assertNotEqual(new_sec, init_sec)
|
||||
self.assertEqual(Server.SERVER_STATE["config"]["enrollment_secret"], new_sec)
|
||||
|
||||
# 2. Check DB
|
||||
conn = sqlite3.connect(self.test_db)
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT enrollment_secret FROM license_config WHERE id = 1")
|
||||
db_sec = c.fetchone()[0]
|
||||
conn.close()
|
||||
self.assertEqual(db_sec, new_sec)
|
||||
|
||||
# 3. Check config file
|
||||
with open(self.test_config, "r", encoding="utf-8") as f:
|
||||
cfg_file = json.load(f)
|
||||
self.assertEqual(cfg_file["enrollment_secret"], new_sec)
|
||||
|
||||
# 4. Check export_client_profile automatically generates a new secret
|
||||
resp = Server.export_client_profile(server_host="10.0.0.5", server_port=9443)
|
||||
exported_profile = json.loads(resp.body.decode("utf-8"))
|
||||
self.assertIn("enrollment_secret", exported_profile)
|
||||
self.assertNotEqual(exported_profile["enrollment_secret"], new_sec)
|
||||
self.assertEqual(exported_profile["enrollment_secret"], Server.SERVER_STATE["config"]["enrollment_secret"])
|
||||
|
||||
# 5. Check reset API endpoint
|
||||
reset_res = Server.reset_server_enrollment_secret()
|
||||
self.assertEqual(reset_res["status"], "success")
|
||||
self.assertIn("enrollment_secret", reset_res)
|
||||
self.assertEqual(reset_res["enrollment_secret"], Server.SERVER_STATE["config"]["enrollment_secret"])
|
||||
|
||||
def test_hermes_offline_node_detection_and_metrics(self):
|
||||
"""Tests that Hermes reporting detects silent forwarders (>15m) and includes host telemetry metrics."""
|
||||
Server.init_db(self.test_db)
|
||||
Server.SERVER_STATE["config"] = {
|
||||
"db_path": self.test_db,
|
||||
"evaluation_window_hours": 12
|
||||
}
|
||||
conn = sqlite3.connect(self.test_db)
|
||||
now = datetime.now(timezone.utc)
|
||||
thirty_min_ago = (now - timedelta(minutes=30)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
two_min_ago = (now - timedelta(minutes=2)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
metrics_node1 = json.dumps({"cpu_percent": 15.5, "memory_percent": 60.0, "disk_percent_free": 45.0})
|
||||
metrics_node2 = json.dumps({"cpu_percent": 88.0, "memory_percent": 95.0, "disk_percent_free": 4.5})
|
||||
|
||||
conn.execute("""
|
||||
INSERT INTO clients (client_id, hostname, os_type, cert_fingerprint, status, last_seen, metrics_json, cpu_percent, memory_percent, disk_percent_free)
|
||||
VALUES ('node-online', 'srv-online.corp', 'windows', 'FP1', 'active', ?, ?, 15.5, 60.0, 45.0)
|
||||
""", (two_min_ago, metrics_node1))
|
||||
|
||||
conn.execute("""
|
||||
INSERT INTO clients (client_id, hostname, os_type, cert_fingerprint, status, last_seen, metrics_json, cpu_percent, memory_percent, disk_percent_free)
|
||||
VALUES ('node-dead', 'srv-dead.corp', 'linux', 'FP2', 'active', ?, ?, 88.0, 95.0, 4.5)
|
||||
""", (thirty_min_ago, metrics_node2))
|
||||
|
||||
# Also insert a verified operational error for the online node
|
||||
conn.execute("""
|
||||
INSERT INTO active_issues (fingerprint, site_name, server, signature, severity, message, os_type, first_seen, last_seen, run_count, status, last_run_id)
|
||||
VALUES ('default:srv-online.corp:AppCrash', 'default', 'srv-online.corp', 'AppCrash', 'ERROR', 'Segmentation fault', 'windows', ?, ?, 2, 'VERIFIED', 'run-1')
|
||||
""", (two_min_ago, two_min_ago))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
report = Server.get_verified_anomalies_for_hermes()
|
||||
self.assertGreaterEqual(len(report), 2)
|
||||
|
||||
# Verify offline node alert
|
||||
offline_alert = next((r for r in report if r.get("status") == "NODE_OFFLINE"), None)
|
||||
self.assertIsNotNone(offline_alert)
|
||||
self.assertEqual(offline_alert["server"], "srv-dead.corp")
|
||||
self.assertEqual(offline_alert["severity"], "CRITICAL")
|
||||
self.assertIn("has gone silent", offline_alert["message"])
|
||||
self.assertEqual(offline_alert["metrics"]["disk_percent_free"], 4.5)
|
||||
|
||||
# Verify online error has attached telemetry
|
||||
online_err = next((r for r in report if r.get("signature") == "AppCrash"), None)
|
||||
self.assertIsNotNone(online_err)
|
||||
self.assertEqual(online_err["metrics"]["cpu_percent"], 15.5)
|
||||
|
||||
def test_routine_service_noise_filtering(self):
|
||||
"""Tests that benign routine background service start/stop cycles are filtered from anomaly spam."""
|
||||
self.assertTrue(Server.is_routine_service_noise("sppsvc", "The Software Protection service has entered the stopped state.", "INFO"))
|
||||
self.assertTrue(Server.is_routine_service_noise("edgeupdate", "The Microsoft Edge Update service entered the running state.", "INFO"))
|
||||
self.assertTrue(Server.is_routine_service_noise("Service Control Manager", "The Windows Update service entered the stopped state.", "INFO"))
|
||||
self.assertTrue(Server.is_routine_service_noise("systemd", "Service started successfully.", "INFO"))
|
||||
|
||||
# Errors or warnings must NOT be filtered
|
||||
self.assertFalse(Server.is_routine_service_noise("sppsvc", "Software Protection activation failed with error 0xC004F074", "ERROR"))
|
||||
self.assertFalse(Server.is_routine_service_noise("Database", "Connection pool exhausted", "WARNING"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
"""
|
||||
LOGAR - Unit and Integration Tests for Auto-Update & Update Alerting
|
||||
Tests version comparison, ServerUpdateManager, ClientUpdater, REST endpoints, and WebUI rendering.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import uuid
|
||||
import tempfile
|
||||
import unittest
|
||||
import warnings
|
||||
from typing import Optional
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
# Ensure parent directory and src directory are in path
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "src")))
|
||||
|
||||
import Server
|
||||
from version import LOGAR_VERSION, parse_version, compare_versions, is_newer_version
|
||||
from update_manager import ServerUpdateManager
|
||||
from client_updater import ClientUpdater
|
||||
|
||||
|
||||
class TestUpdateSystem(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
self.test_db = os.path.join(self.test_dir, f"test_db_{uuid.uuid4().hex[:8]}.db")
|
||||
self.test_updates_dir = os.path.join(self.test_dir, "updates")
|
||||
Server.init_db(self.test_db)
|
||||
self.update_mgr = ServerUpdateManager(update_dir=self.test_updates_dir, check_enabled=False)
|
||||
Server.SERVER_STATE["update_manager"] = self.update_mgr
|
||||
Server.SERVER_STATE["config"] = {"db_path": self.test_db}
|
||||
|
||||
def tearDown(self):
|
||||
import shutil
|
||||
if os.path.exists(self.test_dir):
|
||||
try:
|
||||
shutil.rmtree(self.test_dir)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def test_semantic_version_parsing_and_comparison(self):
|
||||
"""Tests robust 4-part semantic version comparison."""
|
||||
self.assertEqual(parse_version("0.3.5.2"), (0, 3, 5, 2))
|
||||
self.assertEqual(parse_version("v0.3.5.2"), (0, 3, 5, 2))
|
||||
self.assertEqual(parse_version("0.3.5"), (0, 3, 5, 0))
|
||||
self.assertEqual(parse_version("1.0.0.0"), (1, 0, 0, 0))
|
||||
self.assertEqual(parse_version(""), (0, 0, 0, 0))
|
||||
|
||||
# Comparisons
|
||||
self.assertEqual(compare_versions("0.3.5.3", "0.3.5.2"), 1)
|
||||
self.assertEqual(compare_versions("0.3.5.2", "0.3.5.2"), 0)
|
||||
self.assertEqual(compare_versions("0.3.5.1", "0.3.5.2"), -1)
|
||||
self.assertEqual(compare_versions("0.4.0.0", "0.3.9.9"), 1)
|
||||
self.assertEqual(compare_versions("1.0.0.0", "0.9.9.9"), 1)
|
||||
|
||||
self.assertTrue(is_newer_version("0.3.6.0", "0.3.5.2"))
|
||||
self.assertFalse(is_newer_version("0.3.5.2", "0.3.5.2"))
|
||||
self.assertFalse(is_newer_version("0.3.5.1", "0.3.5.2"))
|
||||
|
||||
def test_server_update_manager_package_registration(self):
|
||||
"""Tests registering client update packages and calculating SHA256 hashes."""
|
||||
dummy_pkg = os.path.join(self.test_dir, "LOGAR_Client_Setup.exe")
|
||||
with open(dummy_pkg, "wb") as f:
|
||||
f.write(b"LOGAR_WINDOWS_CLIENT_BINARY_PAYLOAD_V0360")
|
||||
|
||||
manifest = self.update_mgr.register_client_package(
|
||||
os_type="windows",
|
||||
file_path=dummy_pkg,
|
||||
version="0.3.6.0",
|
||||
filename="LOGAR_Client_Setup.exe"
|
||||
)
|
||||
self.assertEqual(manifest["os"], "windows")
|
||||
self.assertEqual(manifest["version"], "0.3.6.0")
|
||||
self.assertTrue(len(manifest["sha256"]) == 64)
|
||||
|
||||
# Check hosted packages dictionary
|
||||
packages = self.update_mgr.get_hosted_packages()
|
||||
self.assertTrue(packages["windows"]["exists"])
|
||||
self.assertEqual(packages["windows"]["version"], "0.3.6.0")
|
||||
self.assertEqual(packages["windows"]["sha256"], manifest["sha256"])
|
||||
|
||||
# Check client update evaluation for an older client
|
||||
eval_res = self.update_mgr.check_client_update(
|
||||
client_id="node-test-win",
|
||||
os_type="windows",
|
||||
current_version="0.3.5.2"
|
||||
)
|
||||
self.assertTrue(eval_res["update_available"])
|
||||
self.assertEqual(eval_res["latest_version"], "0.3.6.0")
|
||||
self.assertEqual(eval_res["sha256"], manifest["sha256"])
|
||||
self.assertIn("/api/client/update/download", eval_res["download_url"])
|
||||
|
||||
# Client already on current version -> no update needed
|
||||
eval_current = self.update_mgr.check_client_update(
|
||||
client_id="node-test-win",
|
||||
os_type="windows",
|
||||
current_version="0.3.6.0"
|
||||
)
|
||||
self.assertFalse(eval_current["update_available"])
|
||||
|
||||
def test_client_updater_checksum_validation(self):
|
||||
"""Tests ClientUpdater SHA-256 verification and file integrity validation."""
|
||||
updater = ClientUpdater(config={"server_host": "127.0.0.1", "hermes_port": 8443}, os_type="windows")
|
||||
test_file = os.path.join(self.test_dir, "test_bin.exe")
|
||||
content = b"TEST_PAYLOAD_FOR_AUTO_UPDATER"
|
||||
with open(test_file, "wb") as f:
|
||||
f.write(content)
|
||||
|
||||
import hashlib
|
||||
expected_sha = hashlib.sha256(content).hexdigest()
|
||||
self.assertEqual(updater.calculate_sha256(test_file), expected_sha.lower())
|
||||
|
||||
def test_server_update_endpoints_and_webui(self):
|
||||
"""Tests FastAPI update status, client check, and WebUI rendering with update components."""
|
||||
# 1. Status API
|
||||
status = Server.get_update_status()
|
||||
self.assertIn("current_server_version", status)
|
||||
self.assertIn("hosted_packages", status)
|
||||
|
||||
# 2. Register Windows package
|
||||
dummy_pkg = os.path.join(self.test_dir, "LOGAR_Client_Setup.exe")
|
||||
with open(dummy_pkg, "wb") as f:
|
||||
f.write(b"WIN_CLIENT_V0360")
|
||||
self.update_mgr.register_client_package("windows", dummy_pkg, version="0.3.6.0")
|
||||
|
||||
# 3. Client update check endpoint (Unlicensed should block auto_update)
|
||||
chk_blocked = Server.check_client_update(
|
||||
client_id="win-client-01",
|
||||
os="windows",
|
||||
current_version="0.3.5.1"
|
||||
)
|
||||
self.assertFalse(chk_blocked["update_available"])
|
||||
self.assertIn("not authorized", chk_blocked.get("reason", ""))
|
||||
|
||||
# 4. Enable auto_update feature flag in license config
|
||||
import sqlite3
|
||||
conn = sqlite3.connect(self.test_db)
|
||||
conn.execute("UPDATE license_config SET license_status = 'active', features = '[\"auto_update\"]' WHERE id = 1")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
# 5. Client update check endpoint (Licensed)
|
||||
chk_resp = Server.check_client_update(
|
||||
client_id="win-client-01",
|
||||
os="windows",
|
||||
current_version="0.3.5.1"
|
||||
)
|
||||
self.assertTrue(chk_resp["update_available"])
|
||||
self.assertEqual(chk_resp["latest_version"], "0.3.6.0")
|
||||
|
||||
# 4. Download package endpoint
|
||||
from starlette.requests import Request as StarletteRequest
|
||||
req_scope = {
|
||||
"type": "http",
|
||||
"method": "GET",
|
||||
"path": "/api/client/update/download",
|
||||
"headers": [],
|
||||
"query_string": b"os=windows"
|
||||
}
|
||||
init_req = StarletteRequest(req_scope)
|
||||
dl_resp = Server.download_client_update_package(request=init_req, os="windows")
|
||||
self.assertEqual(dl_resp.status_code, 200)
|
||||
self.assertIn("X-Checksum-SHA256", dl_resp.headers)
|
||||
self.assertIn("ETag", dl_resp.headers)
|
||||
etag = dl_resp.headers["ETag"]
|
||||
|
||||
# 5. Conditional GET with ETag (304 Not Modified)
|
||||
cond_scope = {
|
||||
"type": "http",
|
||||
"method": "GET",
|
||||
"path": "/api/client/update/download",
|
||||
"headers": [(b"if-none-match", etag.encode("utf-8"))],
|
||||
"query_string": b"os=windows"
|
||||
}
|
||||
cached_req = StarletteRequest(cond_scope)
|
||||
cached_resp = Server.download_client_update_package(request=cached_req, os="windows")
|
||||
self.assertEqual(cached_resp.status_code, 304)
|
||||
|
||||
# 6. WebUI rendering verification
|
||||
# Simulate server update available
|
||||
self.update_mgr.server_update_available = True
|
||||
self.update_mgr.latest_upstream_version = "0.3.6.0"
|
||||
|
||||
html = Server.WebUIRenderer.render_dashboard_html(
|
||||
server_state=Server.SERVER_STATE,
|
||||
ingest_queue=Server.INGEST_QUEUE
|
||||
)
|
||||
self.assertIn("LOGAR Server Update Available: v0.3.6.0", html)
|
||||
self.assertIn("tabBtn-updates", html)
|
||||
self.assertIn("tabPane-updates", html)
|
||||
self.assertIn("Client Package Distribution Repository", html)
|
||||
|
||||
def test_update_memoization_and_cache_invalidation(self):
|
||||
"""Tests that package info is served from memory cache and invalidates on update."""
|
||||
dummy_pkg = os.path.join(self.test_dir, "LOGAR_Client_Linux")
|
||||
with open(dummy_pkg, "wb") as f:
|
||||
f.write(b"LINUX_CLIENT_PAYLOAD_V1")
|
||||
|
||||
self.update_mgr.register_client_package("linux", dummy_pkg, version="0.3.6.0")
|
||||
pkgs_1 = self.update_mgr.get_hosted_packages()
|
||||
sha1 = pkgs_1["linux"]["sha256"]
|
||||
|
||||
# Repeated call should hit memory cache with identical hash
|
||||
pkgs_2 = self.update_mgr.get_hosted_packages()
|
||||
self.assertEqual(pkgs_2["linux"]["sha256"], sha1)
|
||||
|
||||
# Invalidate cache on new registration
|
||||
with open(dummy_pkg, "wb") as f:
|
||||
f.write(b"LINUX_CLIENT_PAYLOAD_V2_UPDATED")
|
||||
self.update_mgr.register_client_package("linux", dummy_pkg, version="0.3.7.0")
|
||||
pkgs_3 = self.update_mgr.get_hosted_packages()
|
||||
self.assertEqual(pkgs_3["linux"]["version"], "0.3.7.0")
|
||||
self.assertNotEqual(pkgs_3["linux"]["sha256"], sha1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user