Organize codebase into hub, clients, and common packages with backward compatibility and comprehensive unit test suites
CI Test Suite / Run Component Tests & Pipeline Verification (push) Failing after 1m18s
CI Test Suite / Run Component Tests & Pipeline Verification (push) Failing after 1m18s
This commit is contained in:
+1
-641
@@ -1,641 +1 @@
|
||||
"""
|
||||
LOGAR - Multi-Backend Storage Engine & Data-at-Rest Crypto
|
||||
Provides modular classes for Database schema management, SQLite WAL persistence,
|
||||
PostgreSQL enterprise backend, and OpenPGP / AES-256-GCM data encryption at rest.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import sqlite3
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, Dict, Any, List, Tuple, Callable
|
||||
import pgpy
|
||||
from pgpy.constants import (
|
||||
PubKeyAlgorithm,
|
||||
KeyFlags,
|
||||
HashAlgorithm,
|
||||
SymmetricKeyAlgorithm,
|
||||
CompressionAlgorithm
|
||||
)
|
||||
|
||||
from src.auth_manager import PasswordManager
|
||||
|
||||
|
||||
class StorageCryptoManager:
|
||||
"""Manages OpenPGP keypairs and AES-256-GCM zero-knowledge data-at-rest encryption."""
|
||||
|
||||
@staticmethod
|
||||
def generate_server_keypair(server_name: str) -> Tuple[str, str, str]:
|
||||
"""Generates an OpenPGP RSA 2048 key with encryption capability."""
|
||||
key = pgpy.PGPKey.new(PubKeyAlgorithm.RSAEncryptOrSign, 2048)
|
||||
uid = pgpy.PGPUID.new(server_name)
|
||||
key.add_uid(
|
||||
uid,
|
||||
usage={KeyFlags.EncryptCommunications, KeyFlags.EncryptStorage},
|
||||
hashes=[HashAlgorithm.SHA256, HashAlgorithm.SHA512],
|
||||
ciphers=[SymmetricKeyAlgorithm.AES256, SymmetricKeyAlgorithm.AES128],
|
||||
compression=[
|
||||
CompressionAlgorithm.ZIP,
|
||||
CompressionAlgorithm.ZLIB,
|
||||
CompressionAlgorithm.BZ2,
|
||||
CompressionAlgorithm.Uncompressed
|
||||
]
|
||||
)
|
||||
private_key_armored = str(key)
|
||||
public_key_armored = str(key.pubkey)
|
||||
fingerprint = str(key.pubkey.fingerprint)
|
||||
return private_key_armored, public_key_armored, fingerprint
|
||||
|
||||
@staticmethod
|
||||
def get_server_public_key(server_state: Optional[Dict[str, Any]] = None) -> Optional[pgpy.PGPKey]:
|
||||
"""Retrieves or parses the server's OpenPGP public key for encryption operations."""
|
||||
state = server_state or {}
|
||||
pub_obj = state.get("public_key_obj")
|
||||
if pub_obj:
|
||||
return pub_obj
|
||||
priv_obj = state.get("private_key_obj")
|
||||
if priv_obj and hasattr(priv_obj, "pubkey"):
|
||||
state["public_key_obj"] = priv_obj.pubkey
|
||||
return priv_obj.pubkey
|
||||
pub_blob = state.get("config", {}).get("public_key")
|
||||
if pub_blob:
|
||||
try:
|
||||
key, _ = pgpy.PGPKey.from_blob(pub_blob)
|
||||
state["public_key_obj"] = key
|
||||
return key
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def encrypt_field_for_storage(text: str, force: bool = False, server_state: Optional[Dict[str, Any]] = None, is_unlocked: bool = False) -> str:
|
||||
"""Encrypts a text payload using the server's OpenPGP public key if encrypt_at_rest is enabled."""
|
||||
if not text or not isinstance(text, str):
|
||||
return text
|
||||
state = server_state or {}
|
||||
config = state.get("config", {})
|
||||
should_encrypt = force or (is_unlocked and bool(config.get("encrypt_at_rest", False)))
|
||||
if not should_encrypt:
|
||||
return text
|
||||
|
||||
if text.startswith("-----BEGIN PGP MESSAGE-----"):
|
||||
return text
|
||||
|
||||
pub_key = StorageCryptoManager.get_server_public_key(server_state=state)
|
||||
if not pub_key:
|
||||
return text
|
||||
|
||||
try:
|
||||
msg = pgpy.PGPMessage.new(text)
|
||||
encrypted = pub_key.encrypt(msg)
|
||||
return str(encrypted)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Failed to encrypt field at rest: {e}")
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def decrypt_field_from_storage(text: str, server_state: Optional[Dict[str, Any]] = None) -> str:
|
||||
"""
|
||||
Decrypts an OpenPGP armored message string using the server's private key.
|
||||
Returns cleartext for authorized display and API consumption.
|
||||
"""
|
||||
if not text or not isinstance(text, str):
|
||||
return text
|
||||
if not text.startswith("-----BEGIN PGP MESSAGE-----"):
|
||||
return text
|
||||
|
||||
state = server_state or {}
|
||||
priv_key = state.get("private_key_obj")
|
||||
if not priv_key:
|
||||
priv_blob = state.get("config", {}).get("private_key")
|
||||
if priv_blob:
|
||||
try:
|
||||
priv_key, _ = pgpy.PGPKey.from_blob(priv_blob)
|
||||
state["private_key_obj"] = priv_key
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not priv_key:
|
||||
return text
|
||||
|
||||
try:
|
||||
pgp_msg = pgpy.PGPMessage.from_blob(text)
|
||||
decrypted = priv_key.decrypt(pgp_msg)
|
||||
return str(decrypted.message)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Failed to decrypt field from storage: {e}")
|
||||
return text
|
||||
|
||||
|
||||
class StorageEngine:
|
||||
"""Manages database schema initialization, migrations, multi-engine switching, and telemetry stats."""
|
||||
|
||||
DEFAULT_DB_FILE = "logar_state.db"
|
||||
DEFAULT_ADMIN_USER = os.environ.get("LOGAR_ADMIN_USER", "admin")
|
||||
DEFAULT_ADMIN_PASS = os.environ.get("LOGAR_ADMIN_PASS", "admin_logar_2026!")
|
||||
|
||||
@classmethod
|
||||
def init_db(cls, db_path: str, enrollment_secret: Optional[str] = None, max_seats: int = 0) -> str:
|
||||
"""Initializes the SQLite schema for multi-run temporal tracking, client tracking, users, and license quota."""
|
||||
def _execute_schema(target_path: str):
|
||||
target_dir = os.path.dirname(os.path.abspath(target_path))
|
||||
if target_dir:
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
conn = sqlite3.connect(target_path, timeout=10.0)
|
||||
try:
|
||||
conn.execute("PRAGMA journal_mode=WAL;")
|
||||
conn.execute("PRAGMA busy_timeout=5000;")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 1. Active Issues
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS active_issues (
|
||||
fingerprint TEXT PRIMARY KEY,
|
||||
site_name TEXT,
|
||||
server TEXT,
|
||||
signature TEXT,
|
||||
severity TEXT,
|
||||
message TEXT,
|
||||
os_type TEXT,
|
||||
first_seen TEXT,
|
||||
last_seen TEXT,
|
||||
run_count INTEGER,
|
||||
status TEXT,
|
||||
last_run_id TEXT
|
||||
)
|
||||
""")
|
||||
|
||||
# 2. Ingest Runs
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ingest_runs (
|
||||
run_id TEXT PRIMARY KEY,
|
||||
site_name TEXT,
|
||||
server TEXT,
|
||||
timestamp TEXT,
|
||||
log_count INTEGER
|
||||
)
|
||||
""")
|
||||
|
||||
# 3. License Config
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS license_config (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
max_seats INTEGER NOT NULL DEFAULT 0,
|
||||
enrollment_secret TEXT NOT NULL,
|
||||
license_id TEXT,
|
||||
license_server_url TEXT,
|
||||
issued_at TEXT,
|
||||
expires_at TEXT,
|
||||
license_status TEXT DEFAULT 'unlicensed'
|
||||
)
|
||||
""")
|
||||
|
||||
def _ensure_columns(tbl: str, cols: List[Tuple[str, str]]):
|
||||
try:
|
||||
existing = {row[1] for row in conn.execute(f"PRAGMA table_info({tbl})").fetchall()}
|
||||
for col, col_def in cols:
|
||||
if col not in existing:
|
||||
conn.execute(f"ALTER TABLE {tbl} ADD COLUMN {col} {col_def}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Migrations for license_config
|
||||
_ensure_columns("license_config", [
|
||||
("license_id", "TEXT"),
|
||||
("license_server_url", "TEXT"),
|
||||
("issued_at", "TEXT"),
|
||||
("expires_at", "TEXT"),
|
||||
("license_status", "TEXT DEFAULT 'unlicensed'"),
|
||||
("features", "TEXT DEFAULT '[]'")
|
||||
])
|
||||
|
||||
# 4. Clients Table
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS clients (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
hostname TEXT NOT NULL,
|
||||
os_type TEXT NOT NULL,
|
||||
cert_fingerprint TEXT NOT NULL,
|
||||
status TEXT DEFAULT 'active',
|
||||
first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
# Migrations for clients table
|
||||
_ensure_columns("clients", [
|
||||
("metrics_json", "TEXT DEFAULT '{}'"),
|
||||
("cpu_percent", "REAL DEFAULT 0.0"),
|
||||
("memory_percent", "REAL DEFAULT 0.0"),
|
||||
("disk_percent_free", "REAL DEFAULT 0.0"),
|
||||
("client_version", "TEXT DEFAULT ''"),
|
||||
("update_status", "TEXT DEFAULT 'up_to_date'"),
|
||||
("last_update_check", "TEXT")
|
||||
])
|
||||
|
||||
# 5. Users Table
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
username TEXT PRIMARY KEY,
|
||||
password_hash TEXT NOT NULL,
|
||||
salt TEXT NOT NULL,
|
||||
role TEXT NOT NULL DEFAULT 'admin',
|
||||
created_at TEXT NOT NULL,
|
||||
totp_secret TEXT,
|
||||
totp_enabled INTEGER DEFAULT 0
|
||||
)
|
||||
""")
|
||||
|
||||
# Migrations for users table (2FA)
|
||||
_ensure_columns("users", [
|
||||
("totp_secret", "TEXT"),
|
||||
("totp_enabled", "INTEGER DEFAULT 0")
|
||||
])
|
||||
|
||||
# Ensure default admin user exists
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT COUNT(*) FROM users")
|
||||
if c.fetchone()[0] == 0:
|
||||
now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
pwd_hash, salt = PasswordManager.hash_password(cls.DEFAULT_ADMIN_PASS)
|
||||
conn.execute(
|
||||
"INSERT INTO users (username, password_hash, salt, role, created_at) VALUES (?, ?, ?, 'admin', ?)",
|
||||
(cls.DEFAULT_ADMIN_USER, pwd_hash, salt, now_iso)
|
||||
)
|
||||
|
||||
# 6. Hermes Tokens Table
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS hermes_tokens (
|
||||
token TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
last_used TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'active'
|
||||
)
|
||||
""")
|
||||
|
||||
# Auto-initialize initial Hermes token if table is empty
|
||||
c.execute("SELECT COUNT(*) FROM hermes_tokens")
|
||||
if c.fetchone()[0] == 0:
|
||||
initial_token = f"hermes_live_{secrets.token_hex(20)}"
|
||||
now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
conn.execute(
|
||||
"INSERT INTO hermes_tokens (token, name, created_at, status) VALUES (?, 'Default Hermes Agent', ?, 'active')",
|
||||
(initial_token, now_iso)
|
||||
)
|
||||
|
||||
# 7. Rate Limits & Server Meta Tables
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS rate_limits (
|
||||
ip TEXT NOT NULL,
|
||||
scope TEXT NOT NULL,
|
||||
attempt_time REAL NOT NULL
|
||||
)
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_rate_limits_ip_scope ON rate_limits (ip, scope, attempt_time)")
|
||||
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS server_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
)
|
||||
""")
|
||||
|
||||
status = "active" if max_seats > 0 else "unlicensed"
|
||||
conn.execute("""
|
||||
INSERT OR IGNORE INTO license_config (id, max_seats, enrollment_secret, license_status, features)
|
||||
VALUES (1, ?, ?, ?, '[]')
|
||||
""", (max_seats, enrollment_secret or "", status))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
try:
|
||||
_execute_schema(db_path)
|
||||
return db_path
|
||||
except (sqlite3.OperationalError, PermissionError, OSError) as e:
|
||||
if "readonly" in str(e).lower() or "permission" in str(e).lower() or "locked" in str(e).lower() or isinstance(e, PermissionError):
|
||||
local_app = os.environ.get("LOCALAPPDATA") or os.path.expanduser(r"~\AppData\Local")
|
||||
fallback_dir = os.path.join(local_app, "LOGAR-Server")
|
||||
os.makedirs(fallback_dir, exist_ok=True)
|
||||
fallback_path = os.path.join(fallback_dir, "logar_state.db")
|
||||
if fallback_path != db_path:
|
||||
print(f"[!] Warning: Database path '{db_path}' is not writable ({e}). Falling back to '{fallback_path}'...")
|
||||
_execute_schema(fallback_path)
|
||||
return fallback_path
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def init_postgres_db(pg_config: Dict[str, Any]) -> Tuple[bool, str]:
|
||||
"""Initializes table schema on a PostgreSQL database instance."""
|
||||
try:
|
||||
try:
|
||||
import psycopg2
|
||||
except ImportError:
|
||||
try:
|
||||
import psycopg as psycopg2
|
||||
except ImportError:
|
||||
return False, "PostgreSQL driver not installed on server (install psycopg2-binary or psycopg)"
|
||||
|
||||
host = pg_config.get("host", "localhost")
|
||||
port = int(pg_config.get("port", 5432))
|
||||
database = pg_config.get("database", "logar")
|
||||
user = pg_config.get("user", "postgres")
|
||||
password = pg_config.get("password", "")
|
||||
sslmode = pg_config.get("sslmode", "prefer")
|
||||
timeout = int(pg_config.get("connect_timeout", 5))
|
||||
|
||||
conn = psycopg2.connect(
|
||||
host=host, port=port, dbname=database, user=user, password=password, sslmode=sslmode, connect_timeout=timeout
|
||||
)
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS active_issues (
|
||||
fingerprint VARCHAR(255) PRIMARY KEY,
|
||||
site_name VARCHAR(255),
|
||||
server VARCHAR(255),
|
||||
signature TEXT,
|
||||
severity VARCHAR(50),
|
||||
message TEXT,
|
||||
os_type VARCHAR(50),
|
||||
first_seen VARCHAR(50),
|
||||
last_seen VARCHAR(50),
|
||||
run_count INTEGER,
|
||||
status VARCHAR(50),
|
||||
last_run_id VARCHAR(100)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ingest_runs (
|
||||
run_id VARCHAR(100) PRIMARY KEY,
|
||||
site_name VARCHAR(255),
|
||||
server VARCHAR(255),
|
||||
timestamp VARCHAR(50),
|
||||
log_count INTEGER
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS license_config (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
max_seats INTEGER NOT NULL DEFAULT 0,
|
||||
enrollment_secret TEXT NOT NULL,
|
||||
license_id VARCHAR(100),
|
||||
license_server_url VARCHAR(255),
|
||||
issued_at VARCHAR(50),
|
||||
expires_at VARCHAR(50),
|
||||
license_status VARCHAR(50) DEFAULT 'unlicensed',
|
||||
features TEXT DEFAULT '[]'
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS clients (
|
||||
client_id VARCHAR(100) PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
os_type VARCHAR(50) NOT NULL,
|
||||
cert_fingerprint VARCHAR(255) NOT NULL,
|
||||
status VARCHAR(50) DEFAULT 'active',
|
||||
first_seen VARCHAR(50),
|
||||
last_seen VARCHAR(50)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
username VARCHAR(100) PRIMARY KEY,
|
||||
password_hash TEXT NOT NULL,
|
||||
salt TEXT NOT NULL,
|
||||
role VARCHAR(50) NOT NULL DEFAULT 'admin',
|
||||
created_at VARCHAR(50) NOT NULL,
|
||||
totp_secret TEXT,
|
||||
totp_enabled INTEGER DEFAULT 0
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS hermes_tokens (
|
||||
token VARCHAR(255) PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
created_at VARCHAR(50) NOT NULL,
|
||||
last_used VARCHAR(50),
|
||||
status VARCHAR(50) NOT NULL DEFAULT 'active'
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS rate_limits (
|
||||
ip VARCHAR(100) NOT NULL,
|
||||
scope VARCHAR(50) NOT NULL,
|
||||
attempt_time DOUBLE PRECISION NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS server_meta (
|
||||
key VARCHAR(100) PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
""")
|
||||
# Migrations for Postgres clients table
|
||||
for col, col_def in [
|
||||
("metrics_json", "TEXT DEFAULT '{}'"),
|
||||
("cpu_percent", "DOUBLE PRECISION DEFAULT 0.0"),
|
||||
("memory_percent", "DOUBLE PRECISION DEFAULT 0.0"),
|
||||
("disk_percent_free", "DOUBLE PRECISION DEFAULT 0.0"),
|
||||
("client_version", "VARCHAR(50) DEFAULT ''"),
|
||||
("update_status", "VARCHAR(50) DEFAULT 'up_to_date'"),
|
||||
("last_update_check", "VARCHAR(50)")
|
||||
]:
|
||||
try:
|
||||
cur.execute(f"ALTER TABLE clients ADD COLUMN IF NOT EXISTS {col} {col_def}")
|
||||
except Exception:
|
||||
pass
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return True, "PostgreSQL schema successfully initialized"
|
||||
except Exception as e:
|
||||
return False, f"PostgreSQL connection/init failed: {e}"
|
||||
|
||||
@staticmethod
|
||||
def migrate_sqlite_to_postgres(
|
||||
sqlite_path: str,
|
||||
pg_config: Dict[str, Any],
|
||||
encrypt_fn: Optional[Callable[[str], str]] = None
|
||||
) -> Tuple[bool, str, Dict[str, int]]:
|
||||
"""Transfers all tables and records from SQLite to PostgreSQL with data validation and encryption."""
|
||||
if not os.path.exists(sqlite_path):
|
||||
return False, f"SQLite database not found at {sqlite_path}", {}
|
||||
|
||||
ok, init_msg = StorageEngine.init_postgres_db(pg_config)
|
||||
if not ok:
|
||||
return False, init_msg, {}
|
||||
|
||||
try:
|
||||
try:
|
||||
import psycopg2
|
||||
except ImportError:
|
||||
import psycopg as psycopg2
|
||||
|
||||
host = pg_config.get("host", "localhost")
|
||||
port = int(pg_config.get("port", 5432))
|
||||
database = pg_config.get("database", "logar")
|
||||
user = pg_config.get("user", "postgres")
|
||||
password = pg_config.get("password", "")
|
||||
sslmode = pg_config.get("sslmode", "prefer")
|
||||
timeout = int(pg_config.get("connect_timeout", 5))
|
||||
|
||||
pg_conn = psycopg2.connect(
|
||||
host=host, port=port, dbname=database, user=user, password=password, sslmode=sslmode, connect_timeout=timeout
|
||||
)
|
||||
sqlite_conn = sqlite3.connect(sqlite_path)
|
||||
sqlite_conn.row_factory = sqlite3.Row
|
||||
|
||||
migrated_counts = {}
|
||||
tables = ["active_issues", "ingest_runs", "license_config", "clients", "users", "hermes_tokens", "rate_limits", "server_meta"]
|
||||
|
||||
with pg_conn.cursor() as pg_cur:
|
||||
for tbl in tables:
|
||||
try:
|
||||
s_cur = sqlite_conn.cursor()
|
||||
s_cur.execute(f"SELECT * FROM {tbl}")
|
||||
rows = s_cur.fetchall()
|
||||
if rows:
|
||||
col_names = [col[0] for col in s_cur.description]
|
||||
cols_joined = ", ".join(col_names)
|
||||
placeholders = ", ".join(["%s"] * len(col_names))
|
||||
insert_query = f"INSERT INTO {tbl} ({cols_joined}) VALUES ({placeholders}) ON CONFLICT DO NOTHING;"
|
||||
data = []
|
||||
for r in rows:
|
||||
row_data = [r[c] for c in col_names]
|
||||
if tbl == "active_issues" and "message" in col_names and encrypt_fn:
|
||||
msg_idx = col_names.index("message")
|
||||
row_data[msg_idx] = encrypt_fn(row_data[msg_idx])
|
||||
data.append(row_data)
|
||||
|
||||
pg_cur.executemany(insert_query, data)
|
||||
migrated_counts[tbl] = len(rows)
|
||||
else:
|
||||
migrated_counts[tbl] = 0
|
||||
except Exception as tbl_err:
|
||||
print(f"[!] Warning migrating table {tbl}: {tbl_err}")
|
||||
migrated_counts[tbl] = 0
|
||||
|
||||
pg_conn.commit()
|
||||
|
||||
sqlite_conn.close()
|
||||
pg_conn.close()
|
||||
return True, "Migration completed successfully", migrated_counts
|
||||
except Exception as e:
|
||||
return False, f"Migration failed: {e}", {}
|
||||
|
||||
@staticmethod
|
||||
def get_storage_engine_info(
|
||||
db_path: Optional[str] = None,
|
||||
server_state: Optional[Dict[str, Any]] = None,
|
||||
ingest_queue: Optional[Any] = None,
|
||||
is_unlocked: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
"""Returns metadata about the active storage backend, configuration, and database telemetry."""
|
||||
state = server_state or {}
|
||||
config = state.get("config", {})
|
||||
db_type = config.get("db_type", config.get("storage_engine", "sqlite")).lower()
|
||||
target_db = db_path or config.get("db_path", StorageEngine.DEFAULT_DB_FILE)
|
||||
|
||||
table_stats = {"active_issues": 0, "ingest_runs": 0, "clients": 0, "users": 0}
|
||||
|
||||
licensed_features = []
|
||||
if target_db and os.path.exists(target_db):
|
||||
try:
|
||||
conn_l = sqlite3.connect(target_db, timeout=3.0)
|
||||
row_l = conn_l.execute("SELECT features FROM license_config WHERE id = 1").fetchone()
|
||||
conn_l.close()
|
||||
if row_l and row_l[0]:
|
||||
feat_raw = row_l[0]
|
||||
if isinstance(feat_raw, str):
|
||||
if feat_raw.startswith("["):
|
||||
licensed_features = json.loads(feat_raw)
|
||||
else:
|
||||
licensed_features = [f.strip() for f in feat_raw.split(",") if f.strip()]
|
||||
elif isinstance(feat_raw, list):
|
||||
licensed_features = list(feat_raw)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
flush_interval = ingest_queue.flush_interval if ingest_queue else 2.0
|
||||
batch_size = ingest_queue.batch_size if ingest_queue else 100
|
||||
max_queue = ingest_queue.max_queue if ingest_queue else 50000
|
||||
|
||||
if db_type in ("postgres", "postgresql"):
|
||||
pg_cfg = config.get("postgres", {})
|
||||
host = pg_cfg.get("host", "localhost")
|
||||
port = int(pg_cfg.get("port", 5432))
|
||||
dbname = pg_cfg.get("database", "logar")
|
||||
user = pg_cfg.get("user", "postgres")
|
||||
try:
|
||||
try:
|
||||
import psycopg2
|
||||
except ImportError:
|
||||
import psycopg as psycopg2
|
||||
p_conn = psycopg2.connect(
|
||||
host=host, port=port, dbname=dbname, user=user,
|
||||
password=pg_cfg.get("password", ""),
|
||||
sslmode=pg_cfg.get("sslmode", "prefer"),
|
||||
connect_timeout=int(pg_cfg.get("connect_timeout", 3))
|
||||
)
|
||||
with p_conn.cursor() as p_cur:
|
||||
for tbl in ["active_issues", "ingest_runs", "clients", "users"]:
|
||||
try:
|
||||
p_cur.execute(f"SELECT COUNT(*) FROM {tbl}")
|
||||
r = p_cur.fetchone()
|
||||
table_stats[tbl] = r[0] if r else 0
|
||||
except Exception:
|
||||
pass
|
||||
p_conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
"engine": "postgres",
|
||||
"engine_type": "postgres",
|
||||
"backend_display": "PostgreSQL Enterprise Pool",
|
||||
"connection_target": f"{user}@{host}:{port}/{dbname}",
|
||||
"status": "Enterprise Connected",
|
||||
"table_stats": table_stats,
|
||||
"is_enterprise": True,
|
||||
"feature_unlocked": is_unlocked,
|
||||
"licensed_features": licensed_features,
|
||||
"cache_mode": config.get("cache_mode", "direct"),
|
||||
"encrypt_at_rest": config.get("encrypt_at_rest", False),
|
||||
"flush_interval": flush_interval,
|
||||
"batch_size": batch_size,
|
||||
"max_queue": max_queue,
|
||||
"postgres_config": pg_cfg
|
||||
}
|
||||
else:
|
||||
size_bytes = 0
|
||||
if target_db and os.path.exists(target_db):
|
||||
try:
|
||||
size_bytes = os.path.getsize(target_db)
|
||||
conn = sqlite3.connect(target_db, timeout=3.0)
|
||||
for tbl in ["active_issues", "ingest_runs", "clients", "users"]:
|
||||
try:
|
||||
row = conn.execute(f"SELECT COUNT(*) FROM {tbl}").fetchone()
|
||||
table_stats[tbl] = row[0] if row else 0
|
||||
except Exception:
|
||||
pass
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
size_kb = round(size_bytes / 1024, 2)
|
||||
return {
|
||||
"engine": "sqlite",
|
||||
"engine_type": "sqlite",
|
||||
"backend_display": "SQLite 3 (WAL Mode)",
|
||||
"connection_target": target_db,
|
||||
"db_size_kb": size_kb,
|
||||
"status": "Local WAL Optimized",
|
||||
"table_stats": table_stats,
|
||||
"is_enterprise": False,
|
||||
"feature_unlocked": is_unlocked,
|
||||
"licensed_features": licensed_features,
|
||||
"cache_mode": config.get("cache_mode", "direct"),
|
||||
"encrypt_at_rest": config.get("encrypt_at_rest", False),
|
||||
"flush_interval": flush_interval,
|
||||
"batch_size": batch_size,
|
||||
"max_queue": max_queue,
|
||||
"postgres_config": config.get("postgres", {
|
||||
"host": "localhost",
|
||||
"port": 5432,
|
||||
"database": "logar",
|
||||
"user": "postgres",
|
||||
"password": "",
|
||||
"sslmode": "prefer",
|
||||
"connect_timeout": 5,
|
||||
"pool_size": 10
|
||||
})
|
||||
}
|
||||
from src.hub.storage_engine import *
|
||||
Reference in New Issue
Block a user