Move build and release tools (package_dist.py, upload_release.py, requirements.txt) into compilation/ directory
This commit is contained in:
@@ -0,0 +1,206 @@
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
import zipapp
|
||||
import hashlib
|
||||
import platform
|
||||
import subprocess
|
||||
import argparse
|
||||
|
||||
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
|
||||
SRC_DIR = os.path.join(ROOT_DIR, "src")
|
||||
DIST_DIR = os.path.abspath("dist")
|
||||
BUILD_TEMP = os.path.abspath("build_temp")
|
||||
|
||||
def clean_and_prep():
|
||||
if os.path.exists(DIST_DIR):
|
||||
shutil.rmtree(DIST_DIR)
|
||||
os.makedirs(DIST_DIR, exist_ok=True)
|
||||
if os.path.exists(BUILD_TEMP):
|
||||
shutil.rmtree(BUILD_TEMP)
|
||||
os.makedirs(BUILD_TEMP, exist_ok=True)
|
||||
|
||||
def build_pyinstaller_binary(script_path, binary_name):
|
||||
print(f"[*] Compiling {binary_name} with PyInstaller...")
|
||||
cmd = [
|
||||
sys.executable, "-m", "PyInstaller",
|
||||
"--onefile",
|
||||
"--clean",
|
||||
"--distpath", DIST_DIR,
|
||||
"--workpath", os.path.join(BUILD_TEMP, f"work_{binary_name}"),
|
||||
"--specpath", os.path.join(BUILD_TEMP, f"spec_{binary_name}"),
|
||||
"--name", binary_name,
|
||||
script_path
|
||||
]
|
||||
res = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if res.returncode != 0:
|
||||
print(f"[!] Compilation error for {binary_name}:\n{res.stderr}")
|
||||
raise RuntimeError(f"Failed to build {binary_name}")
|
||||
print(f"[+] Successfully compiled {binary_name}")
|
||||
|
||||
def calculate_sha256(filepath):
|
||||
h = hashlib.sha256()
|
||||
with open(filepath, "rb") as f:
|
||||
while chunk := f.read(65536):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
def write_checksum_file(filename, digest, binary_filename):
|
||||
out_path = os.path.join(DIST_DIR, filename)
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
f.write(f"{digest} {binary_filename}\n")
|
||||
print(f"[+] Generated checksum file: {filename} ({digest[:16]}...)")
|
||||
|
||||
def build_linux_zipapp_fallback():
|
||||
print("[*] Packaging Linux standalone zipapp fallback binaries...")
|
||||
# Linux Client zipapp
|
||||
app_dir = os.path.join(BUILD_TEMP, "linux_app")
|
||||
os.makedirs(app_dir, exist_ok=True)
|
||||
shutil.copy(os.path.join(SRC_DIR, "Linux_Client.py"), os.path.join(app_dir, "Linux_Client.py"))
|
||||
client_out = os.path.join(DIST_DIR, "Linux_Client.bin")
|
||||
zipapp.create_archive(
|
||||
source=app_dir,
|
||||
target=client_out,
|
||||
interpreter="/usr/bin/env python3",
|
||||
main="Linux_Client:main"
|
||||
)
|
||||
# Server zipapp
|
||||
srv_dir = os.path.join(BUILD_TEMP, "linux_srv")
|
||||
os.makedirs(srv_dir, exist_ok=True)
|
||||
shutil.copy(os.path.join(SRC_DIR, "Server.py"), os.path.join(srv_dir, "Server.py"))
|
||||
server_out = os.path.join(DIST_DIR, "Server.bin")
|
||||
zipapp.create_archive(
|
||||
source=srv_dir,
|
||||
target=server_out,
|
||||
interpreter="/usr/bin/env python3",
|
||||
main="Server:main"
|
||||
)
|
||||
|
||||
def build_windows():
|
||||
print("[*] Compiling Windows standalone executables...")
|
||||
win_client_script = os.path.join(SRC_DIR, "Win_Client.py")
|
||||
build_pyinstaller_binary(win_client_script, "Win_Client")
|
||||
|
||||
server_script = os.path.join(SRC_DIR, "Server.py")
|
||||
build_pyinstaller_binary(server_script, "Server")
|
||||
|
||||
client_bin = os.path.join(DIST_DIR, "Win_Client.exe")
|
||||
server_bin = os.path.join(DIST_DIR, "Server.exe")
|
||||
|
||||
sums = []
|
||||
if os.path.exists(client_bin):
|
||||
client_hash = calculate_sha256(client_bin)
|
||||
write_checksum_file("win_client_sha256sum", client_hash, "Win_Client.exe")
|
||||
write_checksum_file("win_agent_sha256sum", client_hash, "Win_Client.exe")
|
||||
sums.append(f"{client_hash} Win_Client.exe")
|
||||
else:
|
||||
print(f"[!] Warning: Expected {client_bin} was not found.")
|
||||
|
||||
if os.path.exists(server_bin):
|
||||
server_hash = calculate_sha256(server_bin)
|
||||
write_checksum_file("win_server_sha256sum", server_hash, "Server.exe")
|
||||
sums.append(f"{server_hash} Server.exe")
|
||||
else:
|
||||
print(f"[!] Warning: Expected {server_bin} was not found.")
|
||||
|
||||
sums_path = os.path.join(DIST_DIR, "SHA256SUMS_windows.txt")
|
||||
with open(sums_path, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(sums) + "\n")
|
||||
|
||||
def build_linux():
|
||||
print("[*] Compiling Linux standalone binaries...")
|
||||
is_linux_host = platform.system() == "Linux"
|
||||
|
||||
if is_linux_host:
|
||||
linux_client_script = os.path.join(SRC_DIR, "Linux_Client.py")
|
||||
build_pyinstaller_binary(linux_client_script, "Linux_Client.bin")
|
||||
|
||||
server_script = os.path.join(SRC_DIR, "Server.py")
|
||||
build_pyinstaller_binary(server_script, "Server.bin")
|
||||
|
||||
# Normalize extensions in case PyInstaller dropped .bin
|
||||
for name in ["Linux_Client", "Server"]:
|
||||
plain_path = os.path.join(DIST_DIR, name)
|
||||
bin_path = os.path.join(DIST_DIR, f"{name}.bin")
|
||||
if os.path.exists(plain_path) and not os.path.exists(bin_path):
|
||||
os.rename(plain_path, bin_path)
|
||||
|
||||
# Ensure executable permissions on Linux
|
||||
for b in ["Linux_Client.bin", "Server.bin"]:
|
||||
p = os.path.join(DIST_DIR, b)
|
||||
if os.path.exists(p):
|
||||
try:
|
||||
os.chmod(p, 0o755)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
print("[!] Note: Host platform is not Linux. Generating executable zipapps for Linux target.")
|
||||
build_linux_zipapp_fallback()
|
||||
|
||||
client_bin = os.path.join(DIST_DIR, "Linux_Client.bin")
|
||||
server_bin = os.path.join(DIST_DIR, "Server.bin")
|
||||
|
||||
sums = []
|
||||
if os.path.exists(client_bin):
|
||||
client_hash = calculate_sha256(client_bin)
|
||||
write_checksum_file("linux_client_sha256sum", client_hash, "Linux_Client.bin")
|
||||
write_checksum_file("linux_agent_sha256sum", client_hash, "Linux_Client.bin")
|
||||
sums.append(f"{client_hash} Linux_Client.bin")
|
||||
else:
|
||||
print(f"[!] Warning: Expected {client_bin} was not found.")
|
||||
|
||||
if os.path.exists(server_bin):
|
||||
server_hash = calculate_sha256(server_bin)
|
||||
write_checksum_file("linux_server_sha256sum", server_hash, "Server.bin")
|
||||
sums.append(f"{server_hash} Server.bin")
|
||||
else:
|
||||
print(f"[!] Warning: Expected {server_bin} was not found.")
|
||||
|
||||
sums_path = os.path.join(DIST_DIR, "SHA256SUMS_linux.txt")
|
||||
with open(sums_path, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(sums) + "\n")
|
||||
|
||||
def main(target=None):
|
||||
if target is None:
|
||||
parser = argparse.ArgumentParser(description="LOGAR Standalone Binary Compiler & Packager")
|
||||
parser.add_argument(
|
||||
"--target", "-t",
|
||||
choices=["windows", "win", "linux", "auto"],
|
||||
default="auto",
|
||||
help="Target platform to compile binaries for (default: auto-detect)"
|
||||
)
|
||||
args, _ = parser.parse_known_args()
|
||||
target = args.target
|
||||
|
||||
if target == "auto":
|
||||
target = "windows" if platform.system() == "Windows" else "linux"
|
||||
elif target == "win":
|
||||
target = "windows"
|
||||
|
||||
print("=" * 60)
|
||||
print(" LOGAR Binary Packaging")
|
||||
print(f" Host Platform: {platform.system()} ({platform.machine()})")
|
||||
print(f" Target Platform: {target.upper()}")
|
||||
print("=" * 60)
|
||||
|
||||
clean_and_prep()
|
||||
|
||||
if target == "windows":
|
||||
build_windows()
|
||||
elif target == "linux":
|
||||
build_linux()
|
||||
else:
|
||||
raise ValueError(f"Unsupported target: {target}")
|
||||
|
||||
# Clean temporary build directory
|
||||
if os.path.exists(BUILD_TEMP):
|
||||
shutil.rmtree(BUILD_TEMP, ignore_errors=True)
|
||||
|
||||
print("\n[+] Binary shipping artifacts assembled in 'dist/':")
|
||||
for f in sorted(os.listdir(DIST_DIR)):
|
||||
sz = os.path.getsize(os.path.join(DIST_DIR, f))
|
||||
print(f" - {f} ({sz / (1024*1024):.2f} MB)" if sz > 1024*1024 else f" - {f} ({sz} bytes)")
|
||||
print("=" * 60)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,7 @@
|
||||
pgpy>=0.6.0
|
||||
standard-imghdr>=3.13.0; python_version >= "3.13"
|
||||
cryptography>=42.0.0
|
||||
fastapi>=0.110.0
|
||||
uvicorn>=0.28.0
|
||||
pydantic>=2.6.0
|
||||
pywin32>=306; sys_platform == "win32"
|
||||
@@ -0,0 +1,204 @@
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import argparse
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import mimetypes
|
||||
import package_dist
|
||||
|
||||
import time
|
||||
|
||||
DEFAULT_GITEA_URL = os.environ.get("GITEA_SERVER_URL", "https://gitea.eibl.tech")
|
||||
DEFAULT_REPO = os.environ.get("GITEA_REPOSITORY", "me0nline/LOGAR")
|
||||
|
||||
def get_existing_assets(base_url, repo, release_id, token):
|
||||
url = f"{base_url}/api/v1/repos/{repo}/releases/{release_id}/assets"
|
||||
req = urllib.request.Request(url, headers={"Authorization": f"token {token}", "Accept": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def delete_asset(base_url, repo, release_id, asset_id, token):
|
||||
url = f"{base_url}/api/v1/repos/{repo}/releases/{release_id}/assets/{asset_id}"
|
||||
req = urllib.request.Request(url, method="DELETE", headers={"Authorization": f"token {token}"})
|
||||
try:
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def upload_file_to_release(base_url, repo, release_id, token, file_path, max_retries=3):
|
||||
filename = os.path.basename(file_path)
|
||||
|
||||
# Clean up existing asset with same name if already present
|
||||
existing_assets = get_existing_assets(base_url, repo, release_id, token)
|
||||
for asset in existing_assets:
|
||||
if asset.get("name") == filename:
|
||||
print(f"[*] Removing existing asset '{filename}' (ID: {asset['id']})...")
|
||||
delete_asset(base_url, repo, release_id, asset["id"], token)
|
||||
|
||||
url = f"{base_url}/api/v1/repos/{repo}/releases/{release_id}/assets?name={urllib.parse.quote(filename)}"
|
||||
with open(file_path, "rb") as f:
|
||||
file_bytes = f.read()
|
||||
|
||||
for attempt in range(1, max_retries + 1):
|
||||
req = urllib.request.Request(url, data=file_bytes, method="POST")
|
||||
req.add_header("Authorization", f"token {token}")
|
||||
req.add_header("Content-Type", "application/octet-stream")
|
||||
req.add_header("Accept", "application/json")
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
print(f"[+] Attached {filename} ({len(file_bytes)} bytes) to release.")
|
||||
return data
|
||||
except urllib.error.HTTPError as e:
|
||||
err = e.read().decode("utf-8", errors="ignore")
|
||||
print(f"[!] Attempt {attempt}/{max_retries} - Error uploading {filename}: HTTP {e.code} - {err}")
|
||||
if attempt < max_retries:
|
||||
time.sleep(2 * attempt)
|
||||
else:
|
||||
return None
|
||||
except Exception as ex:
|
||||
print(f"[!] Attempt {attempt}/{max_retries} - Exception uploading {filename}: {ex}")
|
||||
if attempt < max_retries:
|
||||
time.sleep(2 * attempt)
|
||||
else:
|
||||
return None
|
||||
|
||||
def create_or_get_release(base_url, repo, tag, token, title=None, notes=None):
|
||||
url = f"{base_url}/api/v1/repos/{repo}/releases"
|
||||
headers = {
|
||||
"Authorization": f"token {token}",
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json"
|
||||
}
|
||||
|
||||
# Check if release exists
|
||||
check_url = f"{base_url}/api/v1/repos/{repo}/releases/tags/{urllib.parse.quote(tag)}"
|
||||
check_req = urllib.request.Request(check_url, headers={"Authorization": f"token {token}"})
|
||||
try:
|
||||
with urllib.request.urlopen(check_req) as resp:
|
||||
existing = json.loads(resp.read().decode("utf-8"))
|
||||
print(f"[*] Found existing release for tag {tag} (ID: {existing['id']})")
|
||||
if notes or title:
|
||||
patch_url = f"{base_url}/api/v1/repos/{repo}/releases/{existing['id']}"
|
||||
patch_payload = {}
|
||||
if title:
|
||||
patch_payload["name"] = title
|
||||
if notes:
|
||||
patch_payload["body"] = notes
|
||||
patch_req = urllib.request.Request(
|
||||
patch_url,
|
||||
data=json.dumps(patch_payload).encode("utf-8"),
|
||||
headers=headers,
|
||||
method="PATCH"
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(patch_req) as p_resp:
|
||||
print(f"[+] Updated release description for tag {tag}")
|
||||
except Exception as e:
|
||||
print(f"[!] Warning: Could not update existing release description: {e}")
|
||||
return existing["id"]
|
||||
except urllib.error.HTTPError:
|
||||
pass
|
||||
|
||||
# Create new release
|
||||
payload = {
|
||||
"tag_name": tag,
|
||||
"name": title or f"LOGAR Release {tag}",
|
||||
"body": notes or f"Automated binary release for {tag}.",
|
||||
"draft": False,
|
||||
"prerelease": False
|
||||
}
|
||||
req = urllib.request.Request(url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST")
|
||||
try:
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
created = json.loads(resp.read().decode("utf-8"))
|
||||
print(f"[+] Created release {tag} (ID: {created['id']})")
|
||||
return created["id"]
|
||||
except urllib.error.HTTPError as e:
|
||||
print(f"[*] Release creation returned HTTP {e.code}. Checking if peer runner created it concurrently...")
|
||||
for attempt in range(1, 6):
|
||||
time.sleep(2)
|
||||
try:
|
||||
with urllib.request.urlopen(check_req) as resp:
|
||||
existing = json.loads(resp.read().decode("utf-8"))
|
||||
print(f"[+] Retrieved peer-created release for tag {tag} (ID: {existing['id']})")
|
||||
return existing["id"]
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Upload LOGAR compiled binaries directly to Gitea Release")
|
||||
parser.add_argument("--tag", default=os.environ.get("GITEA_REF_NAME"), help="Release tag name (e.g. v1.0.1)")
|
||||
parser.add_argument("--token", default=os.environ.get("GITEA_TOKEN"), help="Gitea Personal Access Token (or set GITEA_TOKEN env var)")
|
||||
parser.add_argument("--url", default=DEFAULT_GITEA_URL, help="Base Gitea instance URL")
|
||||
parser.add_argument("--repo", default=DEFAULT_REPO, help="Repository owner/name")
|
||||
parser.add_argument("--title", default=os.environ.get("RELEASE_TITLE"), help="Release title")
|
||||
parser.add_argument("--notes", default=os.environ.get("RELEASE_NOTES"), help="Release description / notes")
|
||||
parser.add_argument("--notes-file", default=None, help="Path to markdown file with release notes")
|
||||
parser.add_argument("--skip-build", action="store_true", help="Skip running package_dist.py before upload")
|
||||
args = parser.parse_args()
|
||||
|
||||
tag = args.tag
|
||||
if not tag:
|
||||
tag = input("Enter tag name (e.g. v1.0.1): ").strip()
|
||||
|
||||
token = args.token
|
||||
if not token:
|
||||
token = input("Enter Gitea Token: ").strip()
|
||||
|
||||
if not tag or not token:
|
||||
print("[!] Tag and Token are required.")
|
||||
sys.exit(1)
|
||||
|
||||
# Resolve release notes
|
||||
notes = args.notes
|
||||
if not notes and args.notes_file and os.path.exists(args.notes_file):
|
||||
with open(args.notes_file, "r", encoding="utf-8") as nf:
|
||||
notes = nf.read()
|
||||
elif not notes and os.path.exists("RELEASE_NOTES.md"):
|
||||
with open("RELEASE_NOTES.md", "r", encoding="utf-8") as nf:
|
||||
notes = nf.read()
|
||||
elif not notes:
|
||||
notes = (
|
||||
f"## LOGAR Release {tag}\n\n"
|
||||
"### Changes in this Release:\n"
|
||||
"- **Dual Platform Gitea Release Automation**: Added dedicated Windows (`release-windows.yml`) and Linux (`release-linux.yml`) Gitea Actions to compile native executables and publish assets concurrently.\n"
|
||||
"- **Dedicated SHA-256 Checksums**: Release assets now include dedicated checksum files matching `[win/linux]_[client/agent]_sha256sum` (e.g., `win_client_sha256sum`, `win_agent_sha256sum`, `win_server_sha256sum`, `linux_client_sha256sum`, `linux_agent_sha256sum`, `linux_server_sha256sum`).\n"
|
||||
"- **Removed Client Filter Logic**: Removed restrictive source-level noise filtering on edge forwarders. Clients now stream all candidate events from `INFO` up to `ERROR` across the lookback window instead of discarding them at the source.\n"
|
||||
"- **State Tracking & Deduplication**: Added persistent state tracking (`client_state.json`) with cursor and record number deduplication so previously transmitted events are never resent.\n"
|
||||
"- **24-Hour Lookback Window**: Forwarders now scan and upload events from the last 24 hours, skipping older entries.\n"
|
||||
"- **Lightweight Distribution Structure**: Cleaned `out/` to strictly contain deployment documentation and sample configurations.\n"
|
||||
"- **Automated Gitea CI/CD**: Integrated push testing workflow (`ci.yml`) and automated release asset packaging.\n"
|
||||
)
|
||||
|
||||
title = args.title or f"LOGAR Release {tag}"
|
||||
|
||||
if not args.skip_build:
|
||||
print("[*] Assembling compiled binaries...")
|
||||
package_dist.main()
|
||||
|
||||
dist_dir = os.path.abspath("dist")
|
||||
if not os.path.exists(dist_dir) or not os.listdir(dist_dir):
|
||||
print("[!] No binaries found in dist/. Run package_dist.py first.")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"[*] Connecting to Gitea: {args.url} (repo: {args.repo})...")
|
||||
release_id = create_or_get_release(args.url, args.repo, tag, token, title=title, notes=notes)
|
||||
|
||||
print(f"[*] Uploading binary assets from '{dist_dir}'...")
|
||||
for f in sorted(os.listdir(dist_dir)):
|
||||
fpath = os.path.join(dist_dir, f)
|
||||
if os.path.isfile(fpath):
|
||||
upload_file_to_release(args.url, args.repo, release_id, token, fpath)
|
||||
|
||||
print(f"\n[+] Release successfully published with binary assets: {args.url}/{args.repo}/releases/tag/{tag}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user