From a2c8a307f048ba83326ee74ada20c86df36940d4 Mon Sep 17 00:00:00 2001 From: administrateur Date: Tue, 1 Apr 2025 12:23:58 +0200 Subject: [PATCH] feat: added granularity for copying, more results in JSON --- script_creator.py | 56 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/script_creator.py b/script_creator.py index 505a917..b806b11 100644 --- a/script_creator.py +++ b/script_creator.py @@ -59,27 +59,38 @@ encoding = Slash,BackSlash,Colon,Dot log_debug(f"Created new rclone config at {config_path}") return config_path -def rclone_sync(website, local_path): - config_file = write_rclone_config() - remote_path = f"nextcloud:{NEXTCLOUD_PATH}/@{website}" - - cmd = f"rclone --config {config_file} sync {remote_path} {local_path} --use-server-modtime --delete-after --fast-list --multi-thread-streams=4 --transfers=8 --checkers=16 --verbose" - log_debug(f"Syncing from {remote_path} to {local_path}") - return stream_command(cmd) - def stream_command(cmd): log_debug(f"Executing: {cmd}") process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + output = [] for line in process.stdout: print(line.rstrip(), flush=True) + output.append(line.rstrip()) process.wait() - return process.returncode + return process.returncode, "\n".join(output) + +def rclone_sync(website, local_path, file_path=None): + config_file = write_rclone_config() + remote_base = f"nextcloud:{NEXTCLOUD_PATH}/@{website}" + + if file_path: + remote_path = f"{remote_base}/{file_path}" + dest_path = os.path.join(local_path, file_path) + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + cmd = f"rclone --config {config_file} copy {remote_path} {os.path.dirname(dest_path)} --use-server-modtime" + else: + remote_path = remote_base + cmd = f"rclone --config {config_file} sync {remote_path} {local_path} --use-server-modtime --delete-after --fast-list --multi-thread-streams=4 --transfers=8 --checkers=16" + + log_debug(f"Syncing from {remote_path} to {local_path}") + return stream_command(cmd) @app.route("/build", methods=["POST"]) def build_mkdocs(): log_debug("POST request received at /build") data = request.json website = data.get("WEBSITE") + file_path = data.get("FILE") error_callback = data.get("ERROR_CALLBACK", N8N_WEBHOOK_URL) if not website: @@ -104,8 +115,9 @@ def build_mkdocs(): src = os.path.join(tmp_path, "mkdocs.yml") try: - if rclone_sync(website, tmp_path) != 0: - return jsonify({"status": "error", "message": "rclone sync failed"}), 500 + code, output_sync = rclone_sync(website, tmp_path, file_path) + if code != 0: + return jsonify({"status": "error", "message": "rclone sync failed", "stdout": output_sync}), 500 if not os.path.exists(src): log_debug(f"{src} not found after rclone sync") @@ -113,16 +125,26 @@ def build_mkdocs(): log_debug(f"Running MkDocs build: {src} -> {compile_path}") mkdocs_cmd = f"mkdocs build --no-strict --config-file {src} --site-dir {compile_path}" - if stream_command(mkdocs_cmd) != 0: - return jsonify({"status": "error", "message": "MkDocs build failed"}), 500 + code, output_build = stream_command(mkdocs_cmd) + if code != 0: + return jsonify({"status": "error", "message": "MkDocs build failed", "stdout": output_build}), 500 log_debug(f"Performing differential copy from {compile_path} to {final_path}") rsync_cmd = f"rsync -a --delete {compile_path}/ {final_path}/" - if stream_command(rsync_cmd) != 0: - return jsonify({"status": "error", "message": "Rsync failed"}), 500 + code, output_rsync = stream_command(rsync_cmd) + if code != 0: + return jsonify({"status": "error", "message": "Rsync failed", "stdout": output_rsync}), 500 log_debug(f"MkDocs build and sync successful for website: {website}") - return jsonify({"status": "success", "message": "Build successful"}), 200 + return jsonify({ + "status": "success", + "message": "Build successful", + "stdout": { + "rclone": output_sync, + "mkdocs": output_build, + "rsync": output_rsync + } + }), 200 finally: redis_client.delete(lock_key) @@ -130,4 +152,4 @@ def build_mkdocs(): if __name__ == "__main__": log_debug("Starting Flask server on 0.0.0.0:80") - app.run(host="0.0.0.0", port=80) \ No newline at end of file + app.run(host="0.0.0.0", port=80)