diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release-linux.yml similarity index 65% rename from .gitea/workflows/release.yml rename to .gitea/workflows/release-linux.yml index bd4c7af..8c76655 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release-linux.yml @@ -1,19 +1,25 @@ -name: Release Binaries +name: Release Linux Binaries on: push: tags: - 'v*' workflow_dispatch: + inputs: + tag: + description: 'Release tag (e.g. v1.0.1)' + required: false + default: 'v1.0.1' jobs: - release: + release-linux: + name: Build & Release Linux Binaries runs-on: ubuntu-latest steps: - name: Checkout Code uses: actions/checkout@v4 - - name: Install Python and Dependencies + - name: Install Python and Build Dependencies run: | if command -v apt-get >/dev/null 2>&1; then apt-get update -y @@ -22,15 +28,15 @@ jobs: python3 -m pip install --upgrade pip --break-system-packages || python3 -m pip install --upgrade pip || true pip3 install pyinstaller -r requirements.txt --break-system-packages || pip3 install pyinstaller -r requirements.txt - - name: Compile Standalone Binaries + - name: Compile Standalone Linux Binaries run: | - python3 package_dist.py + python3 package_dist.py --target linux - - name: Publish Release + - name: Publish Linux Release Assets env: GITEA_TOKEN: ${{ secrets.TAG_TOKEN || github.token }} GITEA_SERVER_URL: ${{ github.server_url }} GITEA_REPOSITORY: ${{ github.repository }} - GITEA_REF_NAME: ${{ github.ref_name }} + GITEA_REF_NAME: ${{ inputs.tag || github.ref_name }} run: | python3 upload_release.py --skip-build diff --git a/.gitea/workflows/release-windows.yml b/.gitea/workflows/release-windows.yml new file mode 100644 index 0000000..2947b91 --- /dev/null +++ b/.gitea/workflows/release-windows.yml @@ -0,0 +1,65 @@ +name: Release Windows Binaries + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + tag: + description: 'Release tag (e.g. v1.0.1)' + required: false + default: 'v1.0.1' + +jobs: + release-windows: + name: Build & Release Windows Binaries + runs-on: windows-latest + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + continue-on-error: true + + - name: Install Dependencies + shell: powershell + run: | + $py = "python" + if (-not (Get-Command "python" -ErrorAction SilentlyContinue)) { + if (Get-Command "py" -ErrorAction SilentlyContinue) { + $py = "py -3.12" + } + } + & $py -m pip install --upgrade pip + & $py -m pip install pyinstaller -r requirements.txt + + - name: Compile Standalone Windows Binaries + shell: powershell + run: | + $py = "python" + if (-not (Get-Command "python" -ErrorAction SilentlyContinue)) { + if (Get-Command "py" -ErrorAction SilentlyContinue) { + $py = "py -3.12" + } + } + & $py package_dist.py --target windows + + - name: Publish Windows Release Assets + shell: powershell + env: + GITEA_TOKEN: ${{ secrets.TAG_TOKEN || github.token }} + GITEA_SERVER_URL: ${{ github.server_url }} + GITEA_REPOSITORY: ${{ github.repository }} + GITEA_REF_NAME: ${{ inputs.tag || github.ref_name }} + run: | + $py = "python" + if (-not (Get-Command "python" -ErrorAction SilentlyContinue)) { + if (Get-Command "py" -ErrorAction SilentlyContinue) { + $py = "py -3.12" + } + } + & $py upload_release.py --skip-build diff --git a/README.md b/README.md index 6cd0bf3..5e87234 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,8 @@ LOGAR/ ├── .gitea/ │ └── workflows/ │ ├── ci.yml # Continuous Integration automated test suite (runs on every push) -│ └── release.yml # Automated standalone binary release workflow (runs on tag v*) +│ ├── release-linux.yml # Linux release workflow (compiles Server.bin, Linux_Client.bin, checksums) +│ └── release-windows.yml # Windows release workflow (compiles Server.exe, Win_Client.exe, checksums) ├── .gitignore # Ignore venv, caches, DBs, and private keys ├── requirements.txt # Unified dependencies ├── README.md # Comprehensive documentation @@ -308,7 +309,9 @@ Continuous integration is automated via [`.gitea/workflows/ci.yml`](.gitea/workf ## Automated Releases via Gitea Actions -Release builds are automated via [`.gitea/workflows/release.yml`](.gitea/workflows/release.yml) using your Gitea action runner: +Release builds are automated via two dedicated Gitea Actions workflows running concurrently on native platform runners: +- [`.gitea/workflows/release-linux.yml`](.gitea/workflows/release-linux.yml) (`ubuntu-latest`) +- [`.gitea/workflows/release-windows.yml`](.gitea/workflows/release-windows.yml) (`windows-latest`) ### Publishing a Release Whenever you want to release a new version with compiled standalone binaries: @@ -316,22 +319,54 @@ Whenever you want to release a new version with compiled standalone binaries: git tag v1.0.1 git push origin v1.0.1 ``` +*(You can also trigger builds manually via the Gitea UI using the **Run workflow** button (`workflow_dispatch`) on either workflow).* -### What Gitea Actions Does Automatically: -1. Gitea runner executes the workflow on tag push. -2. Installs Python, system build tools (`binutils`, `zip`), PyInstaller, and project dependencies via `apt-get` and `pip3`. -3. Runs `package_dist.py` to compile standalone binaries: - - `Linux_Client.bin` (native ELF binary compiled with PyInstaller) - - `Server.bin` (native server ELF binary compiled with PyInstaller) - - `Win_Client.pyz` (standalone executable zipapp) - - `SHA256SUMS.txt` (SHA-256 cryptographic checksums) -4. Publishes the Gitea release directly via Python (`python3 upload_release.py --skip-build`) using the Gitea REST API to attach the compiled binary assets (avoiding runner Node runtime limitations). +### Automated Multi-Platform Compilation: +1. **Linux Runner** (`release-linux.yml`): + - Compiles native Linux ELF executables: `Linux_Client.bin` and `Server.bin`. + - Generates dedicated SHA-256 checksum files: + - `linux_client_sha256sum` (verification for `Linux_Client.bin`) + - `linux_agent_sha256sum` (alias for client/agent integrations) + - `linux_server_sha256sum` (verification for `Server.bin`) + - `SHA256SUMS_linux.txt` (summary manifest) + - Attaches all Linux assets to the Gitea release. -### Building & Publishing Windows Executables (`.exe`) Locally -Because the Linux Gitea runner compiles ELF binaries, native Windows PE executables (`Win_Client.exe`, `Server.exe`) can be built and published directly from a Windows workstation: +2. **Windows Runner** (`release-windows.yml`): + - Compiles native Windows PE executables: `Win_Client.exe` and `Server.exe`. + - Generates dedicated SHA-256 checksum files: + - `win_client_sha256sum` (verification for `Win_Client.exe`) + - `win_agent_sha256sum` (alias for client/agent integrations) + - `win_server_sha256sum` (verification for `Server.exe`) + - `SHA256SUMS_windows.txt` (summary manifest) + - Attaches all Windows assets to the Gitea release. +3. **Concurrent Publishing & Conflict Handling**: + `upload_release.py` includes automatic retry and conflict resolution so concurrent Windows and Linux runners attach their respective assets to the release without collision. + +### Verifying Checksums +- On Linux: + ```bash + sha256sum -c linux_client_sha256sum + # or + sha256sum -c linux_server_sha256sum + ``` +- On Windows (PowerShell): + ```powershell + Get-FileHash .\Win_Client.exe -Algorithm SHA256 + Get-Content .\win_client_sha256sum + ``` + +### Local Packaging & Manual Upload +You can also compile and package binaries locally anytime: +```bash +# Windows +py -3.12 package_dist.py --target windows + +# Linux +python3 package_dist.py --target linux +``` +To upload local builds directly to Gitea: ```powershell -# Compiles Win_Client.exe, Server.exe, Linux_Client.bin, and uploads to Gitea -python upload_release.py --tag v1.0.0 --token +python upload_release.py --tag v1.0.1 --token ``` *(Environment variables `GITEA_TOKEN`, `GITEA_SERVER_URL`, `GITEA_REPOSITORY`, and `GITEA_REF_NAME` are also supported automatically).* diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 63f9726..94bed66 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,8 +1,10 @@ # LOGAR Release v1.0.1 ### Changes in this Release: +- **Dual Platform Gitea Release Automation**: Added dedicated Windows (`release-windows.yml`) and Linux (`release-linux.yml`) Gitea Actions to compile native platform binaries (`Win_Client.exe` and `Server.exe` on Windows; `Linux_Client.bin` and `Server.bin` on Linux). +- **Dedicated SHA-256 Checksums**: Release assets now include dedicated checksum files matching `[win/linux]_[client/agent]_sha256sum` (`win_client_sha256sum`, `win_agent_sha256sum`, `win_server_sha256sum`, `linux_client_sha256sum`, `linux_agent_sha256sum`, `linux_server_sha256sum`). - **Removed Client Filter Logic**: Removed restrictive source-level noise filtering on edge forwarders. Clients now collect and stream all candidate events from `INFO` up to `ERROR` over the lookback window instead of discarding them at the source. - **State Tracking & Deduplication**: Added persistent client state tracking (`client_state.json`) with journalctl cursors and Windows Event Log record numbers to guarantee that previously transmitted events are never resent. - **24-Hour Lookback Window**: Forwarders now scan and upload events from the last 24 hours (default `--hours 24`), skipping older entries. - **Lightweight Distribution Structure**: Cleaned `out/` to strictly contain deployment documentation and sample configurations. -- **Automated Gitea CI/CD**: Integrated push testing workflow (`ci.yml`) and automated release asset packaging (`release.yml`). +- **Automated Gitea CI/CD**: Integrated push testing workflow (`ci.yml`) and multi-platform release asset packaging. diff --git a/package_dist.py b/package_dist.py index fd2d1bb..12bc64e 100644 --- a/package_dist.py +++ b/package_dist.py @@ -5,10 +5,10 @@ import zipapp import hashlib import platform import subprocess +import argparse ROOT_DIR = os.path.abspath(os.path.dirname(__file__)) DIST_DIR = os.path.abspath("dist") -OUT_DIR = os.path.abspath("out") BUILD_TEMP = os.path.abspath("build_temp") def clean_and_prep(): @@ -37,84 +37,166 @@ def build_pyinstaller_binary(script_path, binary_name): raise RuntimeError(f"Failed to build {binary_name}") print(f"[+] Successfully compiled {binary_name}") -def build_linux_zipapp_binary(): - print("[*] Packaging Linux_Client.bin executable binary...") +def calculate_sha256(filepath): + h = hashlib.sha256() + with open(filepath, "rb") as f: + while chunk := f.read(65536): + h.update(chunk) + return h.hexdigest() + +def write_checksum_file(filename, digest, binary_filename): + out_path = os.path.join(DIST_DIR, filename) + with open(out_path, "w", encoding="utf-8") as f: + f.write(f"{digest} {binary_filename}\n") + print(f"[+] Generated checksum file: {filename} ({digest[:16]}...)") + +def build_linux_zipapp_fallback(): + print("[*] Packaging Linux standalone zipapp fallback binaries...") + # Linux Client zipapp app_dir = os.path.join(BUILD_TEMP, "linux_app") os.makedirs(app_dir, exist_ok=True) shutil.copy(os.path.join(ROOT_DIR, "Linux_Client.py"), os.path.join(app_dir, "Linux_Client.py")) - - bin_output = os.path.join(DIST_DIR, "Linux_Client.bin") + client_out = os.path.join(DIST_DIR, "Linux_Client.bin") zipapp.create_archive( source=app_dir, - target=bin_output, + target=client_out, interpreter="/usr/bin/env python3", main="Linux_Client:main" ) - print(f"[+] Successfully generated {bin_output}") + # Server zipapp + srv_dir = os.path.join(BUILD_TEMP, "linux_srv") + os.makedirs(srv_dir, exist_ok=True) + shutil.copy(os.path.join(ROOT_DIR, "Server.py"), os.path.join(srv_dir, "Server.py")) + server_out = os.path.join(DIST_DIR, "Server.bin") + zipapp.create_archive( + source=srv_dir, + target=server_out, + interpreter="/usr/bin/env python3", + main="Server:main" + ) -def generate_checksums(): - checksum_file = os.path.join(DIST_DIR, "SHA256SUMS.txt") - lines = [] - for fname in sorted(os.listdir(DIST_DIR)): - if fname == "SHA256SUMS.txt": - continue - fpath = os.path.join(DIST_DIR, fname) - if os.path.isfile(fpath): - with open(fpath, "rb") as f: - digest = hashlib.sha256(f.read()).hexdigest() - lines.append(f"{digest} {fname}") - with open(checksum_file, "w", encoding="utf-8") as f: - f.write("\n".join(lines) + "\n") +def build_windows(): + print("[*] Compiling Windows standalone executables...") + win_client_script = os.path.join(ROOT_DIR, "Win_Client.py") + build_pyinstaller_binary(win_client_script, "Win_Client") -def main(): - print("=" * 60) - print(" LOGAR Binary Packaging (Binaries Only)") - print(f" Platform: {platform.system()} ({platform.machine()})") - print("=" * 60) + server_script = os.path.join(ROOT_DIR, "Server.py") + build_pyinstaller_binary(server_script, "Server") - clean_and_prep() + client_bin = os.path.join(DIST_DIR, "Win_Client.exe") + server_bin = os.path.join(DIST_DIR, "Server.exe") - is_windows = platform.system() == "Windows" - - if is_windows: - # Build Windows client executable - win_client_script = os.path.join(ROOT_DIR, "Win_Client.py") - build_pyinstaller_binary(win_client_script, "Win_Client") - - # Build Windows server executable - server_script = os.path.join(ROOT_DIR, "Server.py") - build_pyinstaller_binary(server_script, "Server") - - # Build Linux client standalone binary - build_linux_zipapp_binary() + sums = [] + if os.path.exists(client_bin): + client_hash = calculate_sha256(client_bin) + write_checksum_file("win_client_sha256sum", client_hash, "Win_Client.exe") + write_checksum_file("win_agent_sha256sum", client_hash, "Win_Client.exe") + sums.append(f"{client_hash} Win_Client.exe") else: - # On Linux runner: Build native Linux binaries + print(f"[!] Warning: Expected {client_bin} was not found.") + + if os.path.exists(server_bin): + server_hash = calculate_sha256(server_bin) + write_checksum_file("win_server_sha256sum", server_hash, "Server.exe") + sums.append(f"{server_hash} Server.exe") + else: + print(f"[!] Warning: Expected {server_bin} was not found.") + + sums_path = os.path.join(DIST_DIR, "SHA256SUMS_windows.txt") + with open(sums_path, "w", encoding="utf-8") as f: + f.write("\n".join(sums) + "\n") + +def build_linux(): + print("[*] Compiling Linux standalone binaries...") + is_linux_host = platform.system() == "Linux" + + if is_linux_host: linux_client_script = os.path.join(ROOT_DIR, "Linux_Client.py") build_pyinstaller_binary(linux_client_script, "Linux_Client.bin") server_script = os.path.join(ROOT_DIR, "Server.py") build_pyinstaller_binary(server_script, "Server.bin") - # Also package standalone Windows zipapp executable - win_app_dir = os.path.join(BUILD_TEMP, "win_app") - os.makedirs(win_app_dir, exist_ok=True) - shutil.copy(os.path.join(ROOT_DIR, "Win_Client.py"), os.path.join(win_app_dir, "Win_Client.py")) - win_bin_output = os.path.join(DIST_DIR, "Win_Client.pyz") - zipapp.create_archive( - source=win_app_dir, - target=win_bin_output, - interpreter="/usr/bin/env python3", - main="Win_Client:main" - ) + # Normalize extensions in case PyInstaller dropped .bin + for name in ["Linux_Client", "Server"]: + plain_path = os.path.join(DIST_DIR, name) + bin_path = os.path.join(DIST_DIR, f"{name}.bin") + if os.path.exists(plain_path) and not os.path.exists(bin_path): + os.rename(plain_path, bin_path) - generate_checksums() + # Ensure executable permissions on Linux + for b in ["Linux_Client.bin", "Server.bin"]: + p = os.path.join(DIST_DIR, b) + if os.path.exists(p): + try: + os.chmod(p, 0o755) + except Exception: + pass + else: + print("[!] Note: Host platform is not Linux. Generating executable zipapps for Linux target.") + build_linux_zipapp_fallback() + + client_bin = os.path.join(DIST_DIR, "Linux_Client.bin") + server_bin = os.path.join(DIST_DIR, "Server.bin") + + sums = [] + if os.path.exists(client_bin): + client_hash = calculate_sha256(client_bin) + write_checksum_file("linux_client_sha256sum", client_hash, "Linux_Client.bin") + write_checksum_file("linux_agent_sha256sum", client_hash, "Linux_Client.bin") + sums.append(f"{client_hash} Linux_Client.bin") + else: + print(f"[!] Warning: Expected {client_bin} was not found.") + + if os.path.exists(server_bin): + server_hash = calculate_sha256(server_bin) + write_checksum_file("linux_server_sha256sum", server_hash, "Server.bin") + sums.append(f"{server_hash} Server.bin") + else: + print(f"[!] Warning: Expected {server_bin} was not found.") + + sums_path = os.path.join(DIST_DIR, "SHA256SUMS_linux.txt") + with open(sums_path, "w", encoding="utf-8") as f: + f.write("\n".join(sums) + "\n") + +def main(target=None): + if target is None: + parser = argparse.ArgumentParser(description="LOGAR Standalone Binary Compiler & Packager") + parser.add_argument( + "--target", "-t", + choices=["windows", "win", "linux", "auto"], + default="auto", + help="Target platform to compile binaries for (default: auto-detect)" + ) + args, _ = parser.parse_known_args() + target = args.target + + if target == "auto": + target = "windows" if platform.system() == "Windows" else "linux" + elif target == "win": + target = "windows" + + print("=" * 60) + print(" LOGAR Binary Packaging") + print(f" Host Platform: {platform.system()} ({platform.machine()})") + print(f" Target Platform: {target.upper()}") + print("=" * 60) + + clean_and_prep() + + if target == "windows": + build_windows() + elif target == "linux": + build_linux() + else: + raise ValueError(f"Unsupported target: {target}") # Clean temporary build directory if os.path.exists(BUILD_TEMP): shutil.rmtree(BUILD_TEMP, ignore_errors=True) print("\n[+] Binary shipping artifacts assembled in 'dist/':") - for f in os.listdir(DIST_DIR): + for f in sorted(os.listdir(DIST_DIR)): sz = os.path.getsize(os.path.join(DIST_DIR, f)) print(f" - {f} ({sz / (1024*1024):.2f} MB)" if sz > 1024*1024 else f" - {f} ({sz} bytes)") print("=" * 60) diff --git a/upload_release.py b/upload_release.py index 2af8b4f..d8550be 100644 --- a/upload_release.py +++ b/upload_release.py @@ -7,6 +7,8 @@ import urllib.parse import mimetypes import package_dist +import time + DEFAULT_GITEA_URL = os.environ.get("GITEA_SERVER_URL", "https://gitea.eibl.tech") DEFAULT_REPO = os.environ.get("GITEA_REPOSITORY", "me0nline/LOGAR") @@ -28,7 +30,7 @@ def delete_asset(base_url, repo, release_id, asset_id, token): except Exception: pass -def upload_file_to_release(base_url, repo, release_id, token, file_path): +def upload_file_to_release(base_url, repo, release_id, token, file_path, max_retries=3): filename = os.path.basename(file_path) # Clean up existing asset with same name if already present @@ -42,20 +44,30 @@ def upload_file_to_release(base_url, repo, release_id, token, file_path): with open(file_path, "rb") as f: file_bytes = f.read() - req = urllib.request.Request(url, data=file_bytes, method="POST") - req.add_header("Authorization", f"token {token}") - req.add_header("Content-Type", "application/octet-stream") - req.add_header("Accept", "application/json") + for attempt in range(1, max_retries + 1): + req = urllib.request.Request(url, data=file_bytes, method="POST") + req.add_header("Authorization", f"token {token}") + req.add_header("Content-Type", "application/octet-stream") + req.add_header("Accept", "application/json") - try: - with urllib.request.urlopen(req) as resp: - data = json.loads(resp.read().decode("utf-8")) - print(f"[+] Attached {filename} ({len(file_bytes)} bytes) to release.") - return data - except urllib.error.HTTPError as e: - err = e.read().decode("utf-8", errors="ignore") - print(f"[!] Error uploading {filename}: HTTP {e.code} - {err}") - return None + try: + with urllib.request.urlopen(req) as resp: + data = json.loads(resp.read().decode("utf-8")) + print(f"[+] Attached {filename} ({len(file_bytes)} bytes) to release.") + return data + except urllib.error.HTTPError as e: + err = e.read().decode("utf-8", errors="ignore") + print(f"[!] Attempt {attempt}/{max_retries} - Error uploading {filename}: HTTP {e.code} - {err}") + if attempt < max_retries: + time.sleep(2 * attempt) + else: + return None + except Exception as ex: + print(f"[!] Attempt {attempt}/{max_retries} - Exception uploading {filename}: {ex}") + if attempt < max_retries: + time.sleep(2 * attempt) + else: + return None def create_or_get_release(base_url, repo, tag, token, title=None, notes=None): url = f"{base_url}/api/v1/repos/{repo}/releases" @@ -103,10 +115,23 @@ def create_or_get_release(base_url, repo, tag, token, title=None, notes=None): "prerelease": False } req = urllib.request.Request(url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST") - with urllib.request.urlopen(req) as resp: - created = json.loads(resp.read().decode("utf-8")) - print(f"[+] Created release {tag} (ID: {created['id']})") - return created["id"] + try: + with urllib.request.urlopen(req) as resp: + created = json.loads(resp.read().decode("utf-8")) + print(f"[+] Created release {tag} (ID: {created['id']})") + return created["id"] + except urllib.error.HTTPError as e: + print(f"[*] Release creation returned HTTP {e.code}. Checking if peer runner created it concurrently...") + for attempt in range(1, 6): + time.sleep(2) + try: + with urllib.request.urlopen(check_req) as resp: + existing = json.loads(resp.read().decode("utf-8")) + print(f"[+] Retrieved peer-created release for tag {tag} (ID: {existing['id']})") + return existing["id"] + except Exception: + pass + raise def main(): parser = argparse.ArgumentParser(description="Upload LOGAR compiled binaries directly to Gitea Release") @@ -144,11 +169,13 @@ def main(): notes = ( f"## LOGAR Release {tag}\n\n" "### Changes in this Release:\n" + "- **Dual Platform Gitea Release Automation**: Added dedicated Windows (`release-windows.yml`) and Linux (`release-linux.yml`) Gitea Actions to compile native executables and publish assets concurrently.\n" + "- **Dedicated SHA-256 Checksums**: Release assets now include dedicated checksum files matching `[win/linux]_[client/agent]_sha256sum` (e.g., `win_client_sha256sum`, `win_agent_sha256sum`, `win_server_sha256sum`, `linux_client_sha256sum`, `linux_agent_sha256sum`, `linux_server_sha256sum`).\n" "- **Removed Client Filter Logic**: Removed restrictive source-level noise filtering on edge forwarders. Clients now stream all candidate events from `INFO` up to `ERROR` across the lookback window instead of discarding them at the source.\n" "- **State Tracking & Deduplication**: Added persistent state tracking (`client_state.json`) with cursor and record number deduplication so previously transmitted events are never resent.\n" "- **24-Hour Lookback Window**: Forwarders now scan and upload events from the last 24 hours, skipping older entries.\n" "- **Lightweight Distribution Structure**: Cleaned `out/` to strictly contain deployment documentation and sample configurations.\n" - "- **Automated Gitea CI/CD**: Integrated push testing workflow (`ci.yml`) and automated release asset packaging (`release.yml`).\n" + "- **Automated Gitea CI/CD**: Integrated push testing workflow (`ci.yml`) and automated release asset packaging.\n" ) title = args.title or f"LOGAR Release {tag}"