From 53c5592bc859806907c7f7f79c5e0edb4f08d241 Mon Sep 17 00:00:00 2001 From: administrateur Date: Sat, 29 Mar 2025 17:46:18 +0100 Subject: [PATCH] fix: using webdav in code --- script_creator.py | 114 +++++++++++++++------------------------------- 1 file changed, 37 insertions(+), 77 deletions(-) diff --git a/script_creator.py b/script_creator.py index 1e4d3b5..e996cf3 100644 --- a/script_creator.py +++ b/script_creator.py @@ -4,29 +4,23 @@ import subprocess import base64 import requests import redis -from threading import Thread, Lock -import time +import shutil +from threading import Lock +from webdav3.client import Client app = Flask(__name__) # Environment variables -MINIO_BUCKET = os.getenv("MINIO_BUCKET", "nextcloud") -MINIO_HOST = os.getenv("MINIO_HOST", "minio.minio.svc.cluster.local:9000") -MINIO_REGION = os.getenv("MINIO_REGION", "us-east-1") -MINIO_MOUNT_PATH = "/mnt" +NEXTCLOUD_URL_DAV = os.getenv("NEXTCLOUD_URL_DAV") +NEXTCLOUD_USER = os.getenv("NEXTCLOUD_USER") +NEXTCLOUD_PASSWORD = os.getenv("NEXTCLOUD_PASSWORD") N8N_WEBHOOK_URL = os.getenv("N8N_WEBHOOK_URL", "https://n8n.n8n.svc.kube.ia86.cc/webhook/7950310f-e526-475a-82d1-63818da79339") DEBUG = bool(os.getenv("DEBUG", True)) -S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY") -S3_SECRET_KEY = os.getenv("S3_SECRET_KEY") -if not S3_ACCESS_KEY or not S3_SECRET_KEY: - raise ValueError("❌ ERROR: Missing S3_ACCESS_KEY or S3_SECRET_KEY") - redis_host = os.getenv("REDIS_HOST", "redis.redis.svc.cluster.local") redis_port = int(os.getenv("REDIS_PORT", 6379)) redis_client = redis.Redis(host=redis_host, port=redis_port, decode_responses=True) -S3_UNMOUNT_CHECK_INTERVAL = int(os.getenv("S3_UNMOUNT_CHECK_INTERVAL", 300)) lock_prefix = "lock:mkdocs:" local_lock = Lock() @@ -34,58 +28,25 @@ def log_debug(msg): if DEBUG: print(f"🔍 DEBUG: {msg}", flush=True) -def mount_s3(): - log_debug("Attempting S3 mount...") - with local_lock: - if not os.path.ismount(MINIO_MOUNT_PATH): - log_debug(f"Mount path {MINIO_MOUNT_PATH} not mounted. Mounting now.") - os.makedirs(MINIO_MOUNT_PATH, exist_ok=True) - credentials_file = "/etc/passwd-s3fs" - log_debug(f"Creating credentials file: {credentials_file}") - with open(credentials_file, "w") as f: - f.write(f"{S3_ACCESS_KEY}:{S3_SECRET_KEY}\n") - os.chmod(credentials_file, 0o600) +def sync_from_nextcloud(website, tmp_path): + remote_path = f"/sites/@{website}/" + local_path = tmp_path - cmd = ( - f"s3fs {MINIO_BUCKET} {MINIO_MOUNT_PATH} " - f"-o passwd_file={credentials_file} " - f"-o url=https://{MINIO_HOST} " - "-o use_path_request_style " - "-o allow_other" - ) + if os.path.exists(local_path): + shutil.rmtree(local_path) + os.makedirs(local_path, exist_ok=True) - log_debug(f"Executing S3 mount command: {cmd}") - result = subprocess.run(cmd, shell=True, capture_output=True, text=True) - if result.returncode != 0: - log_debug(f"S3 mount failed: {result.stderr}") - raise RuntimeError(f"S3 mount error: {result.stderr}") - log_debug("S3 mounted successfully.") - else: - log_debug("S3 already mounted, skipping.") + options = { + 'webdav_hostname': NEXTCLOUD_URL_DAV, + 'webdav_login': NEXTCLOUD_USER, + 'webdav_password': NEXTCLOUD_PASSWORD + } -def unmount_s3(): - log_debug("Attempting S3 unmount...") - with local_lock: - if os.path.ismount(MINIO_MOUNT_PATH): - cmd = f"fusermount -u {MINIO_MOUNT_PATH}" - result = subprocess.run(cmd, shell=True, capture_output=True, text=True) - if result.returncode == 0: - log_debug("S3 unmounted successfully.") - else: - log_debug(f"S3 unmount failed: {result.stderr}") - else: - log_debug("No active S3 mount to unmount.") + client = Client(options) -def unmount_checker(): - while True: - log_debug("Periodic check for S3 unmount...") - time.sleep(S3_UNMOUNT_CHECK_INTERVAL) - active_locks = redis_client.keys(f"{lock_prefix}*") - if not active_locks: - log_debug("No active builds detected, proceeding to unmount S3.") - unmount_s3() - else: - log_debug("Active builds detected, keeping S3 mounted.") + log_debug(f"Starting WebDAV sync from '{remote_path}' to '{local_path}'") + client.download_sync(remote_path=remote_path, local_path=local_path) + log_debug("WebDAV sync completed successfully.") @app.route("/build", methods=["POST"]) def build_mkdocs(): @@ -110,24 +71,22 @@ def build_mkdocs(): log_debug(f"Build already active for website: {website}") return jsonify({"status": "busy", "message": f"Build already active: {website}"}), 429 - try: - mount_s3() + tmp_path = f"/tmp/{website}" + compile_path = f"{tmp_path}#compile" + final_path = f"/srv/{website}" - src = f"{MINIO_MOUNT_PATH}/files/sites/@{website}/mkdocs.yml" - tmp = f"/srv/{website}" + try: + sync_from_nextcloud(website, tmp_path) + + src = os.path.join(tmp_path, "mkdocs.yml") log_debug(f"Checking if mkdocs.yml exists at {src}") - timeout = 10 - start_time = time.time() - while not os.path.exists(src): - if time.time() - start_time > timeout: - log_debug(f"Timeout: {src} not found after {timeout}s") - return jsonify({"error": f"{src} not found after mount"}), 404 - log_debug(f"Waiting for {src} to appear...") - time.sleep(0.5) + if not os.path.exists(src): + log_debug(f"{src} not found after sync") + return jsonify({"error": f"{src} not found after sync"}), 404 - log_debug(f"Running MkDocs build: {src} -> {tmp}") - cmd = f"mkdocs build --quiet --no-strict --config-file {src} --site-dir {tmp}" + log_debug(f"Running MkDocs build: {src} -> {compile_path}") + cmd = f"mkdocs build --quiet --no-strict --config-file {src} --site-dir {compile_path}" result = subprocess.run(cmd, shell=True, capture_output=True, text=True) if result.returncode != 0: @@ -137,7 +96,10 @@ def build_mkdocs(): requests.post(error_callback, json=json_payload, headers={"Content-Type": "application/json"}) return jsonify({"status": "error", "message": "Build failed", "error": result.stderr}), 500 - log_debug(f"MkDocs build successful for website: {website}") + log_debug(f"Performing differential copy from {compile_path} to {final_path}") + subprocess.run(f"rsync -a --delete {compile_path}/ {final_path}/", shell=True, check=True) + + log_debug(f"MkDocs build and sync successful for website: {website}") return jsonify({"status": "success", "message": "Build successful"}), 200 finally: @@ -146,6 +108,4 @@ def build_mkdocs(): if __name__ == "__main__": log_debug("Starting Flask server on 0.0.0.0:80") - checker_thread = Thread(target=unmount_checker, daemon=True) - checker_thread.start() app.run(host="0.0.0.0", port=80)