feat(db): complete SQL management UI, cache modes, PostgreSQL params, and OpenPGP data encryption at rest
This commit is contained in:
+519
-103
@@ -162,6 +162,8 @@ class DbConnectionTestRequest(BaseModel):
|
||||
user: str = "postgres"
|
||||
password: str = ""
|
||||
sslmode: str = "prefer"
|
||||
connect_timeout: int = 5
|
||||
pool_size: int = 10
|
||||
|
||||
|
||||
class DbSwitchBackendRequest(BaseModel):
|
||||
@@ -172,9 +174,21 @@ class DbSwitchBackendRequest(BaseModel):
|
||||
user: Optional[str] = "postgres"
|
||||
password: Optional[str] = ""
|
||||
sslmode: Optional[str] = "prefer"
|
||||
connect_timeout: Optional[int] = 5
|
||||
pool_size: Optional[int] = 10
|
||||
db_path: Optional[str] = None
|
||||
|
||||
|
||||
class DbFullConfigRequest(BaseModel):
|
||||
storage_engine: str = "sqlite" # "sqlite" | "postgres"
|
||||
cache_mode: str = "direct" # "direct" | "buffered"
|
||||
flush_interval: float = 2.0
|
||||
batch_size: int = 100
|
||||
max_queue: int = 50000
|
||||
encrypt_at_rest: bool = False
|
||||
pg_config: Optional[DbConnectionTestRequest] = None
|
||||
|
||||
|
||||
class IngestQueueManager:
|
||||
"""
|
||||
High-throughput in-memory buffering and batch processor for incoming telemetry and log anomalies.
|
||||
@@ -1204,10 +1218,90 @@ def is_feature_licensed(feature: str, db_path: Optional[str] = None) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def get_storage_engine_info(db_path: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Returns metadata about the active storage backend and database telemetry."""
|
||||
def get_server_public_key() -> Optional[pgpy.PGPKey]:
|
||||
"""Retrieves or parses the server's OpenPGP public key for encryption operations."""
|
||||
pub_obj = SERVER_STATE.get("public_key_obj")
|
||||
if pub_obj:
|
||||
return pub_obj
|
||||
priv_obj = SERVER_STATE.get("private_key_obj")
|
||||
if priv_obj and hasattr(priv_obj, "pubkey"):
|
||||
SERVER_STATE["public_key_obj"] = priv_obj.pubkey
|
||||
return priv_obj.pubkey
|
||||
pub_blob = SERVER_STATE.get("config", {}).get("public_key")
|
||||
if pub_blob:
|
||||
try:
|
||||
key, _ = pgpy.PGPKey.from_blob(pub_blob)
|
||||
SERVER_STATE["public_key_obj"] = key
|
||||
return key
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def encrypt_field_for_storage(text: str, force: bool = False) -> str:
|
||||
"""
|
||||
Encrypts a text payload using the server's OpenPGP public key if encrypt_at_rest is enabled.
|
||||
Guarantees zero-knowledge database hosting on external database servers.
|
||||
"""
|
||||
if not text or not isinstance(text, str):
|
||||
return text
|
||||
config = SERVER_STATE.get("config", {})
|
||||
db_type = config.get("db_type", "sqlite").lower()
|
||||
should_encrypt = force or config.get("encrypt_at_rest", False)
|
||||
if not should_encrypt:
|
||||
return text
|
||||
|
||||
if text.startswith("-----BEGIN PGP MESSAGE-----"):
|
||||
return text
|
||||
|
||||
pub_key = get_server_public_key()
|
||||
if not pub_key:
|
||||
return text
|
||||
|
||||
try:
|
||||
msg = pgpy.PGPMessage.new(text)
|
||||
encrypted = pub_key.encrypt(msg)
|
||||
return str(encrypted)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Failed to encrypt field at rest: {e}")
|
||||
return text
|
||||
|
||||
|
||||
def decrypt_field_from_storage(text: str) -> str:
|
||||
"""
|
||||
Decrypts an OpenPGP armored message string using the server's private key.
|
||||
Returns cleartext for authorized display and API consumption.
|
||||
"""
|
||||
if not text or not isinstance(text, str):
|
||||
return text
|
||||
if not text.startswith("-----BEGIN PGP MESSAGE-----"):
|
||||
return text
|
||||
|
||||
priv_key = SERVER_STATE.get("private_key_obj")
|
||||
if not priv_key:
|
||||
priv_blob = SERVER_STATE.get("config", {}).get("private_key")
|
||||
if priv_blob:
|
||||
try:
|
||||
priv_key, _ = pgpy.PGPKey.from_blob(priv_blob)
|
||||
SERVER_STATE["private_key_obj"] = priv_key
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not priv_key:
|
||||
return text
|
||||
|
||||
try:
|
||||
pgp_msg = pgpy.PGPMessage.from_blob(text)
|
||||
decrypted = priv_key.decrypt(pgp_msg)
|
||||
return str(decrypted.message)
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Failed to decrypt field from storage: {e}")
|
||||
return text
|
||||
|
||||
|
||||
def get_storage_engine_info(db_path: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Returns metadata about the active storage backend, configuration, and database telemetry."""
|
||||
config = SERVER_STATE.get("config", {})
|
||||
db_type = config.get("db_type", config.get("storage_engine", "sqlite")).lower()
|
||||
target_db = db_path or config.get("db_path", DEFAULT_DB_FILE)
|
||||
|
||||
table_stats = {"active_issues": 0, "ingest_runs": 0, "clients": 0, "users": 0}
|
||||
@@ -1234,9 +1328,32 @@ def get_storage_engine_info(db_path: Optional[str] = None) -> Dict[str, Any]:
|
||||
if db_type in ("postgres", "postgresql"):
|
||||
pg_cfg = config.get("postgres", {})
|
||||
host = pg_cfg.get("host", "localhost")
|
||||
port = pg_cfg.get("port", 5432)
|
||||
port = int(pg_cfg.get("port", 5432))
|
||||
dbname = pg_cfg.get("database", "logar")
|
||||
user = pg_cfg.get("user", "postgres")
|
||||
try:
|
||||
try:
|
||||
import psycopg2
|
||||
except ImportError:
|
||||
import psycopg as psycopg2
|
||||
p_conn = psycopg2.connect(
|
||||
host=host, port=port, dbname=dbname, user=user,
|
||||
password=pg_cfg.get("password", ""),
|
||||
sslmode=pg_cfg.get("sslmode", "prefer"),
|
||||
connect_timeout=int(pg_cfg.get("connect_timeout", 3))
|
||||
)
|
||||
with p_conn.cursor() as p_cur:
|
||||
for tbl in ["active_issues", "ingest_runs", "clients", "users"]:
|
||||
try:
|
||||
p_cur.execute(f"SELECT COUNT(*) FROM {tbl}")
|
||||
r = p_cur.fetchone()
|
||||
table_stats[tbl] = r[0] if r else 0
|
||||
except Exception:
|
||||
pass
|
||||
p_conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
"engine": "postgres",
|
||||
"engine_type": "postgres",
|
||||
@@ -1246,7 +1363,13 @@ def get_storage_engine_info(db_path: Optional[str] = None) -> Dict[str, Any]:
|
||||
"table_stats": table_stats,
|
||||
"is_enterprise": True,
|
||||
"feature_unlocked": is_unlocked,
|
||||
"licensed_features": licensed_features
|
||||
"licensed_features": licensed_features,
|
||||
"cache_mode": config.get("cache_mode", "direct"),
|
||||
"encrypt_at_rest": config.get("encrypt_at_rest", False),
|
||||
"flush_interval": INGEST_QUEUE.flush_interval,
|
||||
"batch_size": INGEST_QUEUE.batch_size,
|
||||
"max_queue": INGEST_QUEUE.max_queue,
|
||||
"postgres_config": pg_cfg
|
||||
}
|
||||
else:
|
||||
size_bytes = 0
|
||||
@@ -1274,7 +1397,22 @@ def get_storage_engine_info(db_path: Optional[str] = None) -> Dict[str, Any]:
|
||||
"table_stats": table_stats,
|
||||
"is_enterprise": False,
|
||||
"feature_unlocked": is_unlocked,
|
||||
"licensed_features": licensed_features
|
||||
"licensed_features": licensed_features,
|
||||
"cache_mode": config.get("cache_mode", "direct"),
|
||||
"encrypt_at_rest": config.get("encrypt_at_rest", False),
|
||||
"flush_interval": INGEST_QUEUE.flush_interval,
|
||||
"batch_size": INGEST_QUEUE.batch_size,
|
||||
"max_queue": INGEST_QUEUE.max_queue,
|
||||
"postgres_config": config.get("postgres", {
|
||||
"host": "localhost",
|
||||
"port": 5432,
|
||||
"database": "logar",
|
||||
"user": "postgres",
|
||||
"password": "",
|
||||
"sslmode": "prefer",
|
||||
"connect_timeout": 5,
|
||||
"pool_size": 10
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1295,9 +1433,10 @@ def init_postgres_db(pg_config: Dict[str, Any]) -> Tuple[bool, str]:
|
||||
user = pg_config.get("user", "postgres")
|
||||
password = pg_config.get("password", "")
|
||||
sslmode = pg_config.get("sslmode", "prefer")
|
||||
timeout = int(pg_config.get("connect_timeout", 5))
|
||||
|
||||
conn = psycopg2.connect(
|
||||
host=host, port=port, dbname=database, user=user, password=password, sslmode=sslmode, connect_timeout=5
|
||||
host=host, port=port, dbname=database, user=user, password=password, sslmode=sslmode, connect_timeout=timeout
|
||||
)
|
||||
|
||||
with conn.cursor() as cur:
|
||||
@@ -1377,7 +1516,7 @@ def init_postgres_db(pg_config: Dict[str, Any]) -> Tuple[bool, str]:
|
||||
|
||||
|
||||
def migrate_sqlite_to_postgres(sqlite_path: str, pg_config: Dict[str, Any]) -> Tuple[bool, str, Dict[str, int]]:
|
||||
"""Transfers all tables and records from SQLite to PostgreSQL with data validation."""
|
||||
"""Transfers all tables and records from SQLite to PostgreSQL with data validation and encryption."""
|
||||
if not os.path.exists(sqlite_path):
|
||||
return False, f"SQLite database not found at {sqlite_path}", {}
|
||||
|
||||
@@ -1397,9 +1536,10 @@ def migrate_sqlite_to_postgres(sqlite_path: str, pg_config: Dict[str, Any]) -> T
|
||||
user = pg_config.get("user", "postgres")
|
||||
password = pg_config.get("password", "")
|
||||
sslmode = pg_config.get("sslmode", "prefer")
|
||||
timeout = int(pg_config.get("connect_timeout", 5))
|
||||
|
||||
pg_conn = psycopg2.connect(
|
||||
host=host, port=port, dbname=database, user=user, password=password, sslmode=sslmode, connect_timeout=5
|
||||
host=host, port=port, dbname=database, user=user, password=password, sslmode=sslmode, connect_timeout=timeout
|
||||
)
|
||||
sqlite_conn = sqlite3.connect(sqlite_path)
|
||||
sqlite_conn.row_factory = sqlite3.Row
|
||||
@@ -1418,7 +1558,14 @@ def migrate_sqlite_to_postgres(sqlite_path: str, pg_config: Dict[str, Any]) -> T
|
||||
cols_joined = ", ".join(col_names)
|
||||
placeholders = ", ".join(["%s"] * len(col_names))
|
||||
insert_query = f"INSERT INTO {tbl} ({cols_joined}) VALUES ({placeholders}) ON CONFLICT DO NOTHING;"
|
||||
data = [[r[c] for c in col_names] for r in rows]
|
||||
data = []
|
||||
for r in rows:
|
||||
row_data = [r[c] for c in col_names]
|
||||
if tbl == "active_issues" and "message" in col_names:
|
||||
msg_idx = col_names.index("message")
|
||||
row_data[msg_idx] = encrypt_field_for_storage(row_data[msg_idx])
|
||||
data.append(row_data)
|
||||
|
||||
pg_cur.executemany(insert_query, data)
|
||||
migrated_counts[tbl] = len(rows)
|
||||
else:
|
||||
@@ -1456,6 +1603,7 @@ def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: i
|
||||
"""
|
||||
Evaluates ingested log anomalies and updates temporal run tracking.
|
||||
All anomalies are promoted directly to VERIFIED for Hermes AI reasoning and decision-making.
|
||||
Supports SQLite and external PostgreSQL with optional OpenPGP data encryption at rest.
|
||||
"""
|
||||
client_server = payload.get("server", "unknown-host")
|
||||
site_name = payload.get("site_name") or (client_server.split(".", 1)[1] if "." in client_server else "default")
|
||||
@@ -1464,10 +1612,102 @@ def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: i
|
||||
now = datetime.now(timezone.utc)
|
||||
now_iso = now.isoformat()
|
||||
|
||||
config = SERVER_STATE.get("config", {})
|
||||
db_type = config.get("db_type", config.get("storage_engine", "sqlite")).lower()
|
||||
|
||||
if db_type in ("postgres", "postgresql"):
|
||||
try:
|
||||
try:
|
||||
import psycopg2
|
||||
except ImportError:
|
||||
import psycopg as psycopg2
|
||||
|
||||
pg_cfg = config.get("postgres", {})
|
||||
conn = psycopg2.connect(
|
||||
host=pg_cfg.get("host", "localhost"),
|
||||
port=int(pg_cfg.get("port", 5432)),
|
||||
dbname=pg_cfg.get("database", "logar"),
|
||||
user=pg_cfg.get("user", "postgres"),
|
||||
password=pg_cfg.get("password", ""),
|
||||
sslmode=pg_cfg.get("sslmode", "prefer"),
|
||||
connect_timeout=int(pg_cfg.get("connect_timeout", 5))
|
||||
)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"INSERT INTO ingest_runs (run_id, site_name, server, timestamp, log_count) VALUES (%s, %s, %s, %s, %s)",
|
||||
(run_id, site_name, client_server, now_iso, len(logs))
|
||||
)
|
||||
processed_count = 0
|
||||
promoted_to_verified = 0
|
||||
|
||||
for log in logs:
|
||||
severity = str(log.get("severity", "WARNING")).upper()
|
||||
if severity in ["DEBUG", "TRACE"]:
|
||||
continue
|
||||
|
||||
signature = log.get("signature", "unknown")
|
||||
server = log.get("server", client_server)
|
||||
message = log.get("message", "")
|
||||
os_type = log.get("os_type", "unknown")
|
||||
fp = f"{site_name}:{server}:{signature}"
|
||||
|
||||
stored_message = encrypt_field_for_storage(message)
|
||||
|
||||
cursor.execute(
|
||||
"SELECT run_count, first_seen, last_seen, status, last_run_id FROM active_issues WHERE fingerprint = %s",
|
||||
(fp,)
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
|
||||
if row:
|
||||
run_count, first_seen_str, last_seen_str, current_status, last_run_id = row
|
||||
try:
|
||||
last_seen_dt = datetime.fromisoformat(last_seen_str)
|
||||
except Exception:
|
||||
last_seen_dt = now
|
||||
|
||||
if (now - last_seen_dt) > timedelta(hours=window_hours):
|
||||
new_runs = 1
|
||||
new_first_seen = now_iso
|
||||
else:
|
||||
new_runs = run_count + 1 if last_run_id != run_id else run_count
|
||||
new_first_seen = first_seen_str
|
||||
|
||||
new_status = "VERIFIED"
|
||||
if current_status != "VERIFIED":
|
||||
promoted_to_verified += 1
|
||||
|
||||
cursor.execute("""
|
||||
UPDATE active_issues
|
||||
SET run_count = %s, last_seen = %s, first_seen = %s, status = %s, last_run_id = %s, message = %s, severity = %s
|
||||
WHERE fingerprint = %s
|
||||
""", (new_runs, now_iso, new_first_seen, new_status, run_id, stored_message, severity, fp))
|
||||
else:
|
||||
initial_status = "VERIFIED"
|
||||
promoted_to_verified += 1
|
||||
cursor.execute("""
|
||||
INSERT INTO active_issues
|
||||
(fingerprint, site_name, server, signature, severity, message, os_type, first_seen, last_seen, run_count, status, last_run_id)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""", (fp, site_name, server, signature, severity, stored_message, os_type, now_iso, now_iso, 1, initial_status, run_id))
|
||||
|
||||
processed_count += 1
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return {
|
||||
"status": "success",
|
||||
"run_id": run_id,
|
||||
"processed": processed_count,
|
||||
"promoted_verified": promoted_to_verified
|
||||
}
|
||||
except Exception as pg_err:
|
||||
print(f"[!] Warning: PostgreSQL ingestion failed ({pg_err}), falling back to SQLite...")
|
||||
|
||||
# SQLite standard / fallback path
|
||||
conn = sqlite3.connect(db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Record the batch run
|
||||
cursor.execute(
|
||||
"INSERT INTO ingest_runs (run_id, site_name, server, timestamp, log_count) VALUES (?, ?, ?, ?, ?)",
|
||||
(run_id, site_name, client_server, now_iso, len(logs))
|
||||
@@ -1478,7 +1718,6 @@ def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: i
|
||||
|
||||
for log in logs:
|
||||
severity = str(log.get("severity", "WARNING")).upper()
|
||||
# Edge forwarder filter safeguard: retain INFO to ERROR / CRITICAL; strip verbose debug noise
|
||||
if severity in ["DEBUG", "TRACE"]:
|
||||
continue
|
||||
|
||||
@@ -1488,6 +1727,8 @@ def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: i
|
||||
os_type = log.get("os_type", "unknown")
|
||||
fp = f"{site_name}:{server}:{signature}"
|
||||
|
||||
stored_message = encrypt_field_for_storage(message)
|
||||
|
||||
cursor.execute(
|
||||
"SELECT run_count, first_seen, last_seen, status, last_run_id FROM active_issues WHERE fingerprint = ?",
|
||||
(fp,)
|
||||
@@ -1501,15 +1742,11 @@ def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: i
|
||||
except Exception:
|
||||
last_seen_dt = now
|
||||
|
||||
# 12-hour evaluation window expiry check
|
||||
if (now - last_seen_dt) > timedelta(hours=window_hours):
|
||||
new_runs = 1
|
||||
new_first_seen = now_iso
|
||||
else:
|
||||
if last_run_id != run_id:
|
||||
new_runs = run_count + 1
|
||||
else:
|
||||
new_runs = run_count
|
||||
new_runs = run_count + 1 if last_run_id != run_id else run_count
|
||||
new_first_seen = first_seen_str
|
||||
|
||||
new_status = "VERIFIED"
|
||||
@@ -1520,7 +1757,7 @@ def process_ingested_logs(payload: Dict[str, Any], db_path: str, window_hours: i
|
||||
UPDATE active_issues
|
||||
SET run_count = ?, last_seen = ?, first_seen = ?, status = ?, last_run_id = ?, message = ?, severity = ?
|
||||
WHERE fingerprint = ?
|
||||
""", (new_runs, now_iso, new_first_seen, new_status, run_id, message, severity, fp))
|
||||
""", (new_runs, now_iso, new_first_seen, new_status, run_id, stored_message, severity, fp))
|
||||
else:
|
||||
initial_status = "VERIFIED"
|
||||
promoted_to_verified += 1
|
||||
@@ -1879,12 +2116,32 @@ async def handle_socket_client(reader: asyncio.StreamReader, writer: asyncio.Str
|
||||
if client_id and "server" not in log_payload:
|
||||
log_payload["server"] = client_id
|
||||
|
||||
res = process_ingested_logs(
|
||||
log_payload,
|
||||
db_path=SERVER_STATE["config"]["db_path"],
|
||||
window_hours=SERVER_STATE["config"]["evaluation_window_hours"],
|
||||
min_runs=SERVER_STATE["config"]["min_persistence_runs"]
|
||||
)
|
||||
cache_mode = SERVER_STATE.get("config", {}).get("cache_mode", "direct")
|
||||
if cache_mode == "buffered":
|
||||
INGEST_QUEUE.push(log_payload)
|
||||
if len(INGEST_QUEUE._buffer) >= INGEST_QUEUE.batch_size:
|
||||
res = INGEST_QUEUE.flush(
|
||||
db_path=SERVER_STATE["config"]["db_path"],
|
||||
window_hours=SERVER_STATE["config"]["evaluation_window_hours"],
|
||||
min_runs=SERVER_STATE["config"]["min_persistence_runs"]
|
||||
)
|
||||
else:
|
||||
res = {
|
||||
"status": "success",
|
||||
"buffered": True,
|
||||
"processed": len(log_payload.get("logs", [])),
|
||||
"queue_size": len(INGEST_QUEUE._buffer)
|
||||
}
|
||||
else:
|
||||
res = process_ingested_logs(
|
||||
log_payload,
|
||||
db_path=SERVER_STATE["config"]["db_path"],
|
||||
window_hours=SERVER_STATE["config"]["evaluation_window_hours"],
|
||||
min_runs=SERVER_STATE["config"]["min_persistence_runs"]
|
||||
)
|
||||
with INGEST_QUEUE._lock:
|
||||
INGEST_QUEUE._total_ingested += len(log_payload.get("logs", []))
|
||||
INGEST_QUEUE._total_flushed += len(log_payload.get("logs", []))
|
||||
|
||||
resp_bytes = json.dumps(res).encode("utf-8")
|
||||
writer.write(struct.pack(">I", len(resp_bytes)) + resp_bytes)
|
||||
@@ -2466,7 +2723,7 @@ def get_verified_anomalies_for_hermes(auth: str = Depends(require_hermes_or_admi
|
||||
"server": r[2],
|
||||
"signature": r[3],
|
||||
"severity": r[4],
|
||||
"message": r[5],
|
||||
"message": decrypt_field_from_storage(r[5]),
|
||||
"os_type": r[6],
|
||||
"first_seen": r[7],
|
||||
"last_seen": r[8],
|
||||
@@ -2499,7 +2756,7 @@ def get_all_issues(auth: str = Depends(require_hermes_or_admin_auth)):
|
||||
"server": r[2],
|
||||
"signature": r[3],
|
||||
"severity": r[4],
|
||||
"message": r[5],
|
||||
"message": decrypt_field_from_storage(r[5]),
|
||||
"os_type": r[6],
|
||||
"first_seen": r[7],
|
||||
"last_seen": r[8],
|
||||
@@ -2850,6 +3107,51 @@ def api_db_switch_backend(req: DbSwitchBackendRequest):
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/db/config", dependencies=[Depends(require_admin_auth)])
|
||||
def api_db_update_config(req: DbFullConfigRequest):
|
||||
"""Updates complete database storage engine, cache mode, PostgreSQL parameters, and encryption at rest."""
|
||||
if req.storage_engine == "postgres" and not is_feature_licensed("enterprise_db"):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="External PostgreSQL storage is locked. This capability requires the 'enterprise_db' license feature."
|
||||
)
|
||||
|
||||
config = SERVER_STATE.get("config", {})
|
||||
config["storage_engine"] = req.storage_engine
|
||||
config["db_type"] = req.storage_engine
|
||||
config["cache_mode"] = req.cache_mode
|
||||
config["encrypt_at_rest"] = req.encrypt_at_rest
|
||||
|
||||
INGEST_QUEUE.flush_interval = max(0.5, float(req.flush_interval))
|
||||
INGEST_QUEUE.batch_size = max(1, int(req.batch_size))
|
||||
INGEST_QUEUE.max_queue = max(100, int(req.max_queue))
|
||||
config["flush_interval"] = INGEST_QUEUE.flush_interval
|
||||
config["batch_size"] = INGEST_QUEUE.batch_size
|
||||
config["max_queue"] = INGEST_QUEUE.max_queue
|
||||
|
||||
if req.pg_config:
|
||||
pg_dict = req.pg_config.model_dump()
|
||||
config["postgres"] = pg_dict
|
||||
if req.storage_engine == "postgres":
|
||||
ok, msg = init_postgres_db(pg_dict)
|
||||
if not ok:
|
||||
raise HTTPException(status_code=400, detail=f"PostgreSQL connection/init failed: {msg}")
|
||||
|
||||
cfg_path = config.get("config_file_path", CONFIG_FILE_NAME)
|
||||
if os.path.exists(cfg_path):
|
||||
try:
|
||||
with open(cfg_path, "w", encoding="utf-8") as f:
|
||||
json.dump(config, f, indent=2)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Database storage engine, cache mode, and encryption settings successfully updated.",
|
||||
"storage": get_storage_engine_info()
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/db/migrate-to-postgres", dependencies=[Depends(require_admin_auth)])
|
||||
def api_db_migrate_to_postgres(req: DbConnectionTestRequest):
|
||||
"""Migrates all telemetry, runs, clients, users, and license configs from SQLite to PostgreSQL."""
|
||||
@@ -2906,7 +3208,12 @@ def serve_webui(request: Request = None):
|
||||
|
||||
# Fetch verified anomalies
|
||||
c.execute("SELECT fingerprint, server, severity, message, run_count, last_seen FROM active_issues WHERE status = 'VERIFIED' ORDER BY last_seen DESC LIMIT 50")
|
||||
anomalies = c.fetchall()
|
||||
anomalies_raw = c.fetchall()
|
||||
anomalies = []
|
||||
for an in anomalies_raw:
|
||||
afp, srv, sev, msg, rc, ls = an
|
||||
decrypted_msg = decrypt_field_from_storage(msg)
|
||||
anomalies.append((afp, srv, sev, decrypted_msg, rc, ls))
|
||||
|
||||
# Fetch hermes tokens
|
||||
c.execute("SELECT token, name, created_at, last_used, status FROM hermes_tokens ORDER BY created_at DESC")
|
||||
@@ -3036,6 +3343,10 @@ def serve_webui(request: Request = None):
|
||||
if detected_host in ("0.0.0.0", "localhost", ""):
|
||||
detected_host = socket.gethostname()
|
||||
|
||||
is_encrypt_at_rest = config.get("encrypt_at_rest", False)
|
||||
encrypt_status_label = "🔒 OpenPGP Encrypted" if is_encrypt_at_rest else "🔓 Plaintext Storage"
|
||||
encrypt_status_color = "#34d399" if is_encrypt_at_rest else "#94a3b8"
|
||||
|
||||
if is_db_unlocked:
|
||||
db_tab_pane_html = f"""
|
||||
<!-- Tab 5: Database Engine -->
|
||||
@@ -3044,64 +3355,65 @@ def serve_webui(request: Request = None):
|
||||
<div class="card" style="border-color: rgba(0, 229, 255, 0.3);">
|
||||
<div class="card-header">
|
||||
<div>
|
||||
<span class="card-title" style="color: #00E5FF;">⚡ High-Throughput Ingestion Buffer</span>
|
||||
<div style="font-size: 12px; color: var(--text-muted); margin-top: 2px;">In-memory log buffer decoupling TCP socket ingestion from disk transactions</div>
|
||||
<span class="card-title" style="color: #00E5FF;">💾 Storage Engine & Live Telemetry Metrics</span>
|
||||
<div style="font-size: 12px; color: var(--text-muted); margin-top: 2px;">Centralized high-availability SQL pooling, buffer telemetry, and encryption at rest</div>
|
||||
</div>
|
||||
<button onclick="flushIngestBuffer()" class="btn btn-accent" style="padding: 6px 14px; font-size: 12px;">⚡ Force Flush Buffer Now</button>
|
||||
<span class="badge badge-active">{storage_info['engine'].upper()} BACKEND</span>
|
||||
</div>
|
||||
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 14px; margin-top: 14px;">
|
||||
<div style="background: #0f172a; padding: 14px; border-radius: 8px; border: 1px solid var(--border);">
|
||||
<div class="kpi-sub">Buffered Events in Memory</div>
|
||||
<div class="kpi-value" id="liveQueueSize" style="color: #38bdf8; margin-top: 4px;">{queue_stats['queue_size']} <span style="font-size: 13px; color: var(--text-muted);">events</span></div>
|
||||
<div class="kpi-sub">Active Storage Backend</div>
|
||||
<div style="font-weight: 700; color: #38bdf8; margin-top: 4px; font-size: 15px;">{storage_info['backend_display']}</div>
|
||||
<div style="font-size: 11px; color: var(--text-muted); margin-top: 2px;">{storage_info['status']}</div>
|
||||
</div>
|
||||
<div style="background: #0f172a; padding: 14px; border-radius: 8px; border: 1px solid var(--border);">
|
||||
<div class="kpi-sub">Total Events Ingested</div>
|
||||
<div class="kpi-value" style="color: #34d399; margin-top: 4px;">{queue_stats['total_ingested']}</div>
|
||||
<div class="kpi-sub">Telemetry Records</div>
|
||||
<div class="kpi-value" style="color: #34d399; margin-top: 4px; font-size: 18px;">{storage_info['table_stats']['active_issues']} <span style="font-size: 12px; color: var(--text-muted);">Issues</span> / {storage_info['table_stats']['ingest_runs']} <span style="font-size: 12px; color: var(--text-muted);">Runs</span></div>
|
||||
<div style="font-size: 11px; color: var(--text-muted); margin-top: 2px;">Clients: {storage_info['table_stats']['clients']} | Users: {storage_info['table_stats']['users']}</div>
|
||||
</div>
|
||||
<div style="background: #0f172a; padding: 14px; border-radius: 8px; border: 1px solid var(--border);">
|
||||
<div class="kpi-sub">Total Flushed to Storage</div>
|
||||
<div class="kpi-value" style="color: #c084fc; margin-top: 4px;">{queue_stats['total_flushed']}</div>
|
||||
<div class="kpi-sub">Buffered Events in RAM</div>
|
||||
<div class="kpi-value" id="liveQueueSize" style="color: #c084fc; margin-top: 4px; font-size: 18px;">{queue_stats['queue_size']} <span style="font-size: 12px; color: var(--text-muted);">events</span></div>
|
||||
<div style="font-size: 11px; color: var(--text-muted); margin-top: 2px;">Ingested: {queue_stats['total_ingested']} | Flushed: {queue_stats['total_flushed']}</div>
|
||||
</div>
|
||||
<div style="background: #0f172a; padding: 14px; border-radius: 8px; border: 1px solid var(--border);">
|
||||
<div class="kpi-sub">Avg Batch Flush Latency</div>
|
||||
<div class="kpi-value" style="color: #fbbf24; margin-top: 4px;">{queue_stats['avg_flush_latency_ms']} ms</div>
|
||||
<div class="kpi-sub">Data-at-Rest Security</div>
|
||||
<div style="font-weight: 700; color: {encrypt_status_color}; margin-top: 4px; font-size: 15px;">{encrypt_status_label}</div>
|
||||
<div style="font-size: 11px; color: var(--text-muted); margin-top: 2px;">Zero-Knowledge Storage on External DB</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Active Storage Engine Status -->
|
||||
<!-- Storage Engine & SQL Configurator Form -->
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
<span class="card-title">Active Database Storage Backend</span>
|
||||
<span class="badge badge-active">{storage_info['engine']}</span>
|
||||
<span class="card-title">Database Storage Engine Configuration</span>
|
||||
<span class="badge badge-os">STORAGE SELECTION</span>
|
||||
</div>
|
||||
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 14px; margin-bottom: 20px;">
|
||||
<div>
|
||||
<div class="kpi-sub">Engine Backend Type</div>
|
||||
<div style="font-weight: 700; margin-top: 2px;">{storage_info['engine']}</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="kpi-sub">Active Target / Path</div>
|
||||
<div style="font-weight: 700; margin-top: 2px; font-family: 'JetBrains Mono', monospace; font-size: 12px; word-break: break-all;">{storage_info['connection_target']}</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="kpi-sub">Status</div>
|
||||
<div style="font-weight: 700; color: #34d399; margin-top: 2px;">{storage_info['status']}</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="kpi-sub">Telemetry Records</div>
|
||||
<div style="font-weight: 700; margin-top: 2px;">{storage_info['table_stats']['active_issues']} Issues / {storage_info['table_stats']['ingest_runs']} Runs</div>
|
||||
|
||||
<div style="margin-bottom: 20px;">
|
||||
<label class="form-label" style="font-size: 14px; font-weight: 700; margin-bottom: 8px;">Select Storage Engine Backend:</label>
|
||||
<div style="display: flex; gap: 16px; flex-wrap: wrap;">
|
||||
<label style="display: inline-flex; align-items: center; gap: 8px; cursor: pointer; padding: 12px 18px; background: #0f172a; border: 1px solid var(--border); border-radius: 8px;">
|
||||
<input type="radio" name="dbEngineRadio" id="engineRadioSqlite" value="sqlite" {'checked' if storage_info['engine'] == 'sqlite' else ''} onchange="toggleStorageEngineForm()">
|
||||
<span>💾 <strong>Local SQLite 3 (WAL Mode)</strong> <span style="color: var(--text-muted); font-size: 12px;">(Embedded, Zero Configuration)</span></span>
|
||||
</label>
|
||||
<label style="display: inline-flex; align-items: center; gap: 8px; cursor: pointer; padding: 12px 18px; background: #0f172a; border: 1px solid var(--border); border-radius: 8px;">
|
||||
<input type="radio" name="dbEngineRadio" id="engineRadioPostgres" value="postgres" {'checked' if storage_info['engine'] == 'postgres' else ''} onchange="toggleStorageEngineForm()">
|
||||
<span>🐘 <strong>External PostgreSQL Database Server</strong> <span style="color: #34d399; font-size: 12px; font-weight: 600;">(Enterprise Clustering)</span></span>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- PostgreSQL Configuration Panel -->
|
||||
<div style="background: #0f172a; padding: 20px; border-radius: 10px; border: 1px solid var(--border); margin-top: 10px;">
|
||||
<div style="font-size: 14px; font-weight: 700; margin-bottom: 14px; display: flex; align-items: center; gap: 8px;">
|
||||
<span>🐘</span> Enterprise PostgreSQL Connection Configurator
|
||||
<!-- External PostgreSQL Parameters -->
|
||||
<div id="pgConfigSection" style="background: #0f172a; padding: 20px; border-radius: 10px; border: 1px solid var(--border); margin-bottom: 20px; display: {'block' if storage_info['engine'] == 'postgres' else 'none'};">
|
||||
<div style="font-size: 14px; font-weight: 700; margin-bottom: 14px; display: flex; align-items: center; justify-content: space-between;">
|
||||
<span style="display: flex; align-items: center; gap: 8px;">🐘 External PostgreSQL Connection Parameters</span>
|
||||
<span class="badge badge-active">ENTERPRISE POOL</span>
|
||||
</div>
|
||||
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 12px; margin-bottom: 16px;">
|
||||
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 14px; margin-bottom: 16px;">
|
||||
<div class="form-group" style="margin-bottom: 0;">
|
||||
<label class="form-label">Host / IP</label>
|
||||
<label class="form-label">Host / IP Address</label>
|
||||
<input type="text" id="pgHost" class="form-control" value="{pg_config.get('host', 'localhost')}" placeholder="localhost">
|
||||
</div>
|
||||
<div class="form-group" style="margin-bottom: 0;">
|
||||
@@ -3123,21 +3435,85 @@ def serve_webui(request: Request = None):
|
||||
<div class="form-group" style="margin-bottom: 0;">
|
||||
<label class="form-label">SSL Mode</label>
|
||||
<select id="pgSslMode" class="form-control">
|
||||
<option value="prefer">prefer</option>
|
||||
<option value="require">require</option>
|
||||
<option value="verify-ca">verify-ca</option>
|
||||
<option value="verify-full">verify-full</option>
|
||||
<option value="disable">disable</option>
|
||||
<option value="prefer" {'selected' if pg_config.get('sslmode') == 'prefer' else ''}>prefer</option>
|
||||
<option value="require" {'selected' if pg_config.get('sslmode') == 'require' else ''}>require</option>
|
||||
<option value="verify-ca" {'selected' if pg_config.get('sslmode') == 'verify-ca' else ''}>verify-ca</option>
|
||||
<option value="verify-full" {'selected' if pg_config.get('sslmode') == 'verify-full' else ''}>verify-full</option>
|
||||
<option value="disable" {'selected' if pg_config.get('sslmode') == 'disable' else ''}>disable</option>
|
||||
<option value="allow" {'selected' if pg_config.get('sslmode') == 'allow' else ''}>allow</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-group" style="margin-bottom: 0;">
|
||||
<label class="form-label">Connect Timeout (sec)</label>
|
||||
<input type="number" id="pgConnectTimeout" class="form-control" value="{pg_config.get('connect_timeout', 5)}" placeholder="5">
|
||||
</div>
|
||||
<div class="form-group" style="margin-bottom: 0;">
|
||||
<label class="form-label">Connection Pool Size</label>
|
||||
<input type="number" id="pgPoolSize" class="form-control" value="{pg_config.get('pool_size', 10)}" placeholder="10">
|
||||
</div>
|
||||
</div>
|
||||
<div style="display: flex; gap: 10px; flex-wrap: wrap;">
|
||||
<button onclick="testPgConnection()" class="btn btn-secondary">🔍 Test Connection</button>
|
||||
<button onclick="switchStorageEngine('postgres')" class="btn btn-primary">💾 Save & Switch to PostgreSQL</button>
|
||||
<button onclick="migrateToPostgres()" class="btn btn-accent">🚀 Migrate SQLite Data to PostgreSQL</button>
|
||||
<button onclick="switchStorageEngine('sqlite')" class="btn btn-outline">↩️ Switch back to SQLite (WAL)</button>
|
||||
<button type="button" onclick="testPgConnection()" class="btn btn-secondary">🔍 Test Connection</button>
|
||||
<button type="button" onclick="migrateToPostgres()" class="btn btn-accent">🚀 Migrate SQLite Data to PostgreSQL</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Zero-Knowledge OpenPGP Data Encryption at Rest -->
|
||||
<div style="background: #0f172a; padding: 20px; border-radius: 10px; border: 1px solid var(--border); margin-bottom: 20px;">
|
||||
<div style="font-size: 14px; font-weight: 700; margin-bottom: 8px; display: flex; align-items: center; gap: 8px;">
|
||||
<span>🔐</span> Zero-Knowledge Data Encryption at Rest
|
||||
</div>
|
||||
<label style="display: inline-flex; align-items: center; gap: 10px; cursor: pointer; margin-bottom: 8px;">
|
||||
<input type="checkbox" id="encryptAtRestCheck" {'checked' if config.get('encrypt_at_rest', False) else ''} style="width: 18px; height: 18px; accent-color: var(--accent);">
|
||||
<span style="font-weight: 600; font-size: 14px;">Encrypt Sensitive Telemetry & Log Messages with Server OpenPGP Key</span>
|
||||
</label>
|
||||
<p style="font-size: 12px; color: var(--text-muted); line-height: 1.5; margin-left: 28px;">
|
||||
When enabled, anomaly message payloads stored in the external database are encrypted with the server's OpenPGP public key before writing. Only this hub instance can decrypt records for authorized WebUI and Hermes AI usage, ensuring zero-knowledge privacy on remote database servers.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<!-- Telemetry Ingestion Cache Mode & Buffer Tuning -->
|
||||
<div style="background: #0f172a; padding: 20px; border-radius: 10px; border: 1px solid var(--border); margin-bottom: 20px;">
|
||||
<div style="font-size: 14px; font-weight: 700; margin-bottom: 12px; display: flex; align-items: center; justify-content: space-between;">
|
||||
<span style="display: flex; align-items: center; gap: 8px;">⚡ Ingestion Cache & Buffer Mode</span>
|
||||
<button type="button" onclick="flushIngestBuffer()" class="btn btn-accent" style="padding: 4px 10px; font-size: 11px;">🌊 Flush Buffer Now</button>
|
||||
</div>
|
||||
|
||||
<div style="margin-bottom: 14px;">
|
||||
<div style="display: flex; gap: 16px; flex-wrap: wrap;">
|
||||
<label style="display: inline-flex; align-items: center; gap: 8px; cursor: pointer; padding: 8px 14px; background: #151c2e; border: 1px solid var(--border); border-radius: 8px;">
|
||||
<input type="radio" name="cacheModeRadio" id="cacheModeDirect" value="direct" {'checked' if config.get('cache_mode', 'direct') == 'direct' else ''} onchange="toggleCacheModeParams()">
|
||||
<span>⚡ <strong>Immediate Direct Write</strong> <span style="color: var(--text-muted); font-size: 12px;">(Low Latency / Synchronous Verification)</span></span>
|
||||
</label>
|
||||
<label style="display: inline-flex; align-items: center; gap: 8px; cursor: pointer; padding: 8px 14px; background: #151c2e; border: 1px solid var(--border); border-radius: 8px;">
|
||||
<input type="radio" name="cacheModeRadio" id="cacheModeBuffered" value="buffered" {'checked' if config.get('cache_mode', 'direct') == 'buffered' else ''} onchange="toggleCacheModeParams()">
|
||||
<span>🚀 <strong>In-Memory Queue Buffer</strong> <span style="color: var(--text-muted); font-size: 12px;">(High-Throughput Batch Flushing)</span></span>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="bufferParamsSection" style="display: {'grid' if config.get('cache_mode', 'direct') == 'buffered' else 'none'}; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 14px; padding-top: 10px; border-top: 1px solid var(--border);">
|
||||
<div class="form-group" style="margin-bottom: 0;">
|
||||
<label class="form-label">Flush Interval (seconds)</label>
|
||||
<input type="number" id="bufferFlushInterval" step="0.5" class="form-control" value="{config.get('flush_interval', queue_stats.get('flush_interval_seconds', 2.0))}" placeholder="2.0">
|
||||
</div>
|
||||
<div class="form-group" style="margin-bottom: 0;">
|
||||
<label class="form-label">Batch Size Threshold</label>
|
||||
<input type="number" id="bufferBatchSize" class="form-control" value="{config.get('batch_size', queue_stats.get('batch_size_threshold', 100))}" placeholder="100">
|
||||
</div>
|
||||
<div class="form-group" style="margin-bottom: 0;">
|
||||
<label class="form-label">Max Buffer Capacity</label>
|
||||
<input type="number" id="bufferMaxQueue" class="form-control" value="{config.get('max_queue', 50000)}" placeholder="50000">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Global Save Action Bar -->
|
||||
<div style="display: flex; justify-content: flex-end; gap: 12px; margin-top: 20px;">
|
||||
<button type="button" onclick="saveDbConfig()" class="btn btn-primary" style="padding: 10px 24px; font-size: 14px;">
|
||||
💾 Save Database & Cache Settings
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
@@ -3967,6 +4343,70 @@ def serve_webui(request: Request = None):
|
||||
}}
|
||||
|
||||
// --- Storage Engine & Buffer Functions ---
|
||||
function toggleStorageEngineForm() {{
|
||||
const isPg = document.getElementById('engineRadioPostgres') && document.getElementById('engineRadioPostgres').checked;
|
||||
const sec = document.getElementById('pgConfigSection');
|
||||
if (sec) sec.style.display = isPg ? 'block' : 'none';
|
||||
}}
|
||||
|
||||
function toggleCacheModeParams() {{
|
||||
const isBuf = document.getElementById('cacheModeBuffered') && document.getElementById('cacheModeBuffered').checked;
|
||||
const sec = document.getElementById('bufferParamsSection');
|
||||
if (sec) sec.style.display = isBuf ? 'grid' : 'none';
|
||||
}}
|
||||
|
||||
function getFullDbConfigFromForm() {{
|
||||
const isPg = document.getElementById('engineRadioPostgres') && document.getElementById('engineRadioPostgres').checked;
|
||||
const isBuf = document.getElementById('cacheModeBuffered') && document.getElementById('cacheModeBuffered').checked;
|
||||
const encryptAtRest = document.getElementById('encryptAtRestCheck') ? document.getElementById('encryptAtRestCheck').checked : false;
|
||||
|
||||
const flushInterval = document.getElementById('bufferFlushInterval') ? parseFloat(document.getElementById('bufferFlushInterval').value) || 2.0 : 2.0;
|
||||
const batchSize = document.getElementById('bufferBatchSize') ? parseInt(document.getElementById('bufferBatchSize').value) || 100 : 100;
|
||||
const maxQueue = document.getElementById('bufferMaxQueue') ? parseInt(document.getElementById('bufferMaxQueue').value) || 50000 : 50000;
|
||||
|
||||
const pgConfig = {{
|
||||
host: document.getElementById('pgHost') ? document.getElementById('pgHost').value.trim() : 'localhost',
|
||||
port: document.getElementById('pgPort') ? parseInt(document.getElementById('pgPort').value.trim()) || 5432 : 5432,
|
||||
database: document.getElementById('pgDatabase') ? document.getElementById('pgDatabase').value.trim() : 'logar',
|
||||
user: document.getElementById('pgUser') ? document.getElementById('pgUser').value.trim() : 'postgres',
|
||||
password: document.getElementById('pgPassword') ? document.getElementById('pgPassword').value : '',
|
||||
sslmode: document.getElementById('pgSslMode') ? document.getElementById('pgSslMode').value : 'prefer',
|
||||
connect_timeout: document.getElementById('pgConnectTimeout') ? parseInt(document.getElementById('pgConnectTimeout').value.trim()) || 5 : 5,
|
||||
pool_size: document.getElementById('pgPoolSize') ? parseInt(document.getElementById('pgPoolSize').value.trim()) || 10 : 10
|
||||
}};
|
||||
|
||||
return {{
|
||||
storage_engine: isPg ? 'postgres' : 'sqlite',
|
||||
cache_mode: isBuf ? 'buffered' : 'direct',
|
||||
flush_interval: flushInterval,
|
||||
batch_size: batchSize,
|
||||
max_queue: maxQueue,
|
||||
encrypt_at_rest: encryptAtRest,
|
||||
pg_config: pgConfig
|
||||
}};
|
||||
}}
|
||||
|
||||
async function saveDbConfig() {{
|
||||
try {{
|
||||
showAlert('Saving database and cache configuration...');
|
||||
const payload = getFullDbConfigFromForm();
|
||||
const res = await fetch('/api/db/config', {{
|
||||
method: 'POST',
|
||||
headers: {{ 'Content-Type': 'application/json' }},
|
||||
body: JSON.stringify(payload)
|
||||
}});
|
||||
const data = await res.json();
|
||||
if (res.ok && data.status === 'success') {{
|
||||
showAlert('✅ Database and cache settings successfully saved!');
|
||||
setTimeout(() => location.reload(), 1000);
|
||||
}} else {{
|
||||
alert('❌ Save failed: ' + (data.detail || data.message || 'Unknown error'));
|
||||
}}
|
||||
}} catch (err) {{
|
||||
alert('Request failed: ' + err.message);
|
||||
}}
|
||||
}}
|
||||
|
||||
async function flushIngestBuffer() {{
|
||||
try {{
|
||||
showAlert('Flushing in-memory ingestion buffer...');
|
||||
@@ -3990,7 +4430,9 @@ def serve_webui(request: Request = None):
|
||||
database: document.getElementById('pgDatabase') ? document.getElementById('pgDatabase').value.trim() : 'logar',
|
||||
user: document.getElementById('pgUser') ? document.getElementById('pgUser').value.trim() : 'postgres',
|
||||
password: document.getElementById('pgPassword') ? document.getElementById('pgPassword').value : '',
|
||||
sslmode: document.getElementById('pgSslMode') ? document.getElementById('pgSslMode').value : 'prefer'
|
||||
sslmode: document.getElementById('pgSslMode') ? document.getElementById('pgSslMode').value : 'prefer',
|
||||
connect_timeout: document.getElementById('pgConnectTimeout') ? parseInt(document.getElementById('pgConnectTimeout').value.trim()) || 5 : 5,
|
||||
pool_size: document.getElementById('pgPoolSize') ? parseInt(document.getElementById('pgPoolSize').value.trim()) || 10 : 10
|
||||
}};
|
||||
}}
|
||||
|
||||
@@ -4004,7 +4446,7 @@ def serve_webui(request: Request = None):
|
||||
body: JSON.stringify(cfg)
|
||||
}});
|
||||
const data = await res.json();
|
||||
if (res.ok && data.success) {{
|
||||
if (res.ok && data.status === 'success') {{
|
||||
showAlert('✅ PostgreSQL connection successful: ' + data.message);
|
||||
}} else {{
|
||||
alert('❌ PostgreSQL connection test failed: ' + (data.message || data.detail || 'Unknown error'));
|
||||
@@ -4014,36 +4456,10 @@ def serve_webui(request: Request = None):
|
||||
}}
|
||||
}}
|
||||
|
||||
async function switchStorageEngine(engineType) {{
|
||||
try {{
|
||||
const cfg = engineType === 'postgres' ? getPgConfigFromForm() : null;
|
||||
const confirmMsg = engineType === 'postgres'
|
||||
? 'Switch active storage engine to PostgreSQL? Make sure you have migrated your SQLite data if needed.'
|
||||
: 'Switch back to SQLite database storage?';
|
||||
if (!confirm(confirmMsg)) return;
|
||||
|
||||
showAlert('Switching database storage engine to ' + engineType + '...');
|
||||
const res = await fetch('/api/db/switch-backend', {{
|
||||
method: 'POST',
|
||||
headers: {{ 'Content-Type': 'application/json' }},
|
||||
body: JSON.stringify({{ engine: engineType, pg_config: cfg }})
|
||||
}});
|
||||
const data = await res.json();
|
||||
if (res.ok && data.success) {{
|
||||
showAlert('Storage engine switched to ' + engineType + '!');
|
||||
setTimeout(() => location.reload(), 1000);
|
||||
}} else {{
|
||||
alert('Failed to switch engine: ' + (data.message || data.detail || 'Unknown error'));
|
||||
}}
|
||||
}} catch (err) {{
|
||||
alert('Switch engine request failed: ' + err.message);
|
||||
}}
|
||||
}}
|
||||
|
||||
async function migrateToPostgres() {{
|
||||
try {{
|
||||
const cfg = getPgConfigFromForm();
|
||||
if (!confirm('Migrate all data from SQLite database to PostgreSQL? This will transfer existing telemetry, seat records, and user credentials.')) return;
|
||||
if (!confirm('Migrate all data from SQLite database to PostgreSQL? This will transfer existing telemetry, seat records, and user credentials with optional encryption at rest.')) return;
|
||||
|
||||
showAlert('Migrating SQLite data to PostgreSQL...');
|
||||
const res = await fetch('/api/db/migrate-to-postgres', {{
|
||||
@@ -4052,8 +4468,8 @@ def serve_webui(request: Request = None):
|
||||
body: JSON.stringify(cfg)
|
||||
}});
|
||||
const data = await res.json();
|
||||
if (res.ok && data.success) {{
|
||||
showAlert('Migration completed! ' + JSON.stringify(data.migrated_counts));
|
||||
if (res.ok && data.status === 'success') {{
|
||||
showAlert('Migration completed! ' + JSON.stringify(data.counts));
|
||||
setTimeout(() => location.reload(), 1500);
|
||||
}} else {{
|
||||
alert('Migration failed: ' + (data.message || data.detail || 'Error'));
|
||||
|
||||
@@ -682,6 +682,67 @@ class TestServerComponent(unittest.TestCase):
|
||||
# Starlette TestClient may raise RuntimeError when httpx/httpx2 is missing in lightweight runner environments
|
||||
pass
|
||||
|
||||
def test_db_encryption_at_rest_and_full_config_api(self):
|
||||
"""Tests OpenPGP data-at-rest encryption/decryption and full DB/Cache configuration API."""
|
||||
Server.init_db(self.test_db)
|
||||
priv_armored, pub_armored, fp = Server.generate_server_keypair("test-db-node")
|
||||
priv_key, _ = pgpy.PGPKey.from_blob(priv_armored)
|
||||
pub_key, _ = pgpy.PGPKey.from_blob(pub_armored)
|
||||
|
||||
Server.SERVER_STATE["config"] = {
|
||||
"db_path": self.test_db,
|
||||
"storage_engine": "sqlite",
|
||||
"cache_mode": "direct",
|
||||
"encrypt_at_rest": False,
|
||||
"private_key": priv_armored,
|
||||
"public_key": pub_armored,
|
||||
"server_fingerprint": fp
|
||||
}
|
||||
Server.SERVER_STATE["private_key_obj"] = priv_key
|
||||
Server.SERVER_STATE["public_key_obj"] = pub_key
|
||||
|
||||
# 1. Test OpenPGP encryption & decryption roundtrip
|
||||
raw_msg = "Critical database corruption: segment 0x89FA damaged"
|
||||
encrypted_msg = Server.encrypt_field_for_storage(raw_msg, force=True)
|
||||
self.assertTrue(encrypted_msg.startswith("-----BEGIN PGP MESSAGE-----"))
|
||||
self.assertNotIn("Critical database corruption", encrypted_msg)
|
||||
|
||||
decrypted_msg = Server.decrypt_field_from_storage(encrypted_msg)
|
||||
self.assertEqual(decrypted_msg, raw_msg)
|
||||
|
||||
# 2. Test updating config via api_db_update_config (direct -> buffered, encrypt_at_rest = True)
|
||||
cfg_req = Server.DbFullConfigRequest(
|
||||
storage_engine="sqlite",
|
||||
cache_mode="buffered",
|
||||
flush_interval=4.0,
|
||||
batch_size=200,
|
||||
max_queue=25000,
|
||||
encrypt_at_rest=True
|
||||
)
|
||||
res = Server.api_db_update_config(cfg_req)
|
||||
self.assertEqual(res["status"], "success")
|
||||
self.assertEqual(Server.SERVER_STATE["config"]["cache_mode"], "buffered")
|
||||
self.assertTrue(Server.SERVER_STATE["config"]["encrypt_at_rest"])
|
||||
self.assertEqual(Server.INGEST_QUEUE.flush_interval, 4.0)
|
||||
self.assertEqual(Server.INGEST_QUEUE.batch_size, 200)
|
||||
|
||||
# 3. Test that switching to external PostgreSQL requires enterprise_db license
|
||||
pg_req = Server.DbFullConfigRequest(
|
||||
storage_engine="postgres",
|
||||
cache_mode="direct",
|
||||
pg_config=Server.DbConnectionTestRequest(host="db.corp.internal", port=5432, database="logar", user="postgres", password="")
|
||||
)
|
||||
with self.assertRaises(Server.HTTPException) as cm:
|
||||
Server.api_db_update_config(pg_req)
|
||||
self.assertEqual(cm.exception.status_code, 403)
|
||||
self.assertIn("enterprise_db", cm.exception.detail)
|
||||
|
||||
# 4. Verify storage engine info output
|
||||
info = Server.get_storage_engine_info(db_path=self.test_db)
|
||||
self.assertEqual(info["cache_mode"], "buffered")
|
||||
self.assertTrue(info["encrypt_at_rest"])
|
||||
self.assertEqual(info["flush_interval"], 4.0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user