fix: using webdav in code
This commit is contained in:
parent
bc0501e223
commit
53c5592bc8
@ -4,29 +4,23 @@ import subprocess
|
||||
import base64
|
||||
import requests
|
||||
import redis
|
||||
from threading import Thread, Lock
|
||||
import time
|
||||
import shutil
|
||||
from threading import Lock
|
||||
from webdav3.client import Client
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
# Environment variables
|
||||
MINIO_BUCKET = os.getenv("MINIO_BUCKET", "nextcloud")
|
||||
MINIO_HOST = os.getenv("MINIO_HOST", "minio.minio.svc.cluster.local:9000")
|
||||
MINIO_REGION = os.getenv("MINIO_REGION", "us-east-1")
|
||||
MINIO_MOUNT_PATH = "/mnt"
|
||||
NEXTCLOUD_URL_DAV = os.getenv("NEXTCLOUD_URL_DAV")
|
||||
NEXTCLOUD_USER = os.getenv("NEXTCLOUD_USER")
|
||||
NEXTCLOUD_PASSWORD = os.getenv("NEXTCLOUD_PASSWORD")
|
||||
N8N_WEBHOOK_URL = os.getenv("N8N_WEBHOOK_URL", "https://n8n.n8n.svc.kube.ia86.cc/webhook/7950310f-e526-475a-82d1-63818da79339")
|
||||
DEBUG = bool(os.getenv("DEBUG", True))
|
||||
|
||||
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY")
|
||||
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY")
|
||||
if not S3_ACCESS_KEY or not S3_SECRET_KEY:
|
||||
raise ValueError("❌ ERROR: Missing S3_ACCESS_KEY or S3_SECRET_KEY")
|
||||
|
||||
redis_host = os.getenv("REDIS_HOST", "redis.redis.svc.cluster.local")
|
||||
redis_port = int(os.getenv("REDIS_PORT", 6379))
|
||||
redis_client = redis.Redis(host=redis_host, port=redis_port, decode_responses=True)
|
||||
|
||||
S3_UNMOUNT_CHECK_INTERVAL = int(os.getenv("S3_UNMOUNT_CHECK_INTERVAL", 300))
|
||||
lock_prefix = "lock:mkdocs:"
|
||||
local_lock = Lock()
|
||||
|
||||
@ -34,58 +28,25 @@ def log_debug(msg):
|
||||
if DEBUG:
|
||||
print(f"🔍 DEBUG: {msg}", flush=True)
|
||||
|
||||
def mount_s3():
|
||||
log_debug("Attempting S3 mount...")
|
||||
with local_lock:
|
||||
if not os.path.ismount(MINIO_MOUNT_PATH):
|
||||
log_debug(f"Mount path {MINIO_MOUNT_PATH} not mounted. Mounting now.")
|
||||
os.makedirs(MINIO_MOUNT_PATH, exist_ok=True)
|
||||
credentials_file = "/etc/passwd-s3fs"
|
||||
log_debug(f"Creating credentials file: {credentials_file}")
|
||||
with open(credentials_file, "w") as f:
|
||||
f.write(f"{S3_ACCESS_KEY}:{S3_SECRET_KEY}\n")
|
||||
os.chmod(credentials_file, 0o600)
|
||||
def sync_from_nextcloud(website, tmp_path):
|
||||
remote_path = f"/sites/@{website}/"
|
||||
local_path = tmp_path
|
||||
|
||||
cmd = (
|
||||
f"s3fs {MINIO_BUCKET} {MINIO_MOUNT_PATH} "
|
||||
f"-o passwd_file={credentials_file} "
|
||||
f"-o url=https://{MINIO_HOST} "
|
||||
"-o use_path_request_style "
|
||||
"-o allow_other"
|
||||
)
|
||||
if os.path.exists(local_path):
|
||||
shutil.rmtree(local_path)
|
||||
os.makedirs(local_path, exist_ok=True)
|
||||
|
||||
log_debug(f"Executing S3 mount command: {cmd}")
|
||||
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
log_debug(f"S3 mount failed: {result.stderr}")
|
||||
raise RuntimeError(f"S3 mount error: {result.stderr}")
|
||||
log_debug("S3 mounted successfully.")
|
||||
else:
|
||||
log_debug("S3 already mounted, skipping.")
|
||||
options = {
|
||||
'webdav_hostname': NEXTCLOUD_URL_DAV,
|
||||
'webdav_login': NEXTCLOUD_USER,
|
||||
'webdav_password': NEXTCLOUD_PASSWORD
|
||||
}
|
||||
|
||||
def unmount_s3():
|
||||
log_debug("Attempting S3 unmount...")
|
||||
with local_lock:
|
||||
if os.path.ismount(MINIO_MOUNT_PATH):
|
||||
cmd = f"fusermount -u {MINIO_MOUNT_PATH}"
|
||||
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
||||
if result.returncode == 0:
|
||||
log_debug("S3 unmounted successfully.")
|
||||
else:
|
||||
log_debug(f"S3 unmount failed: {result.stderr}")
|
||||
else:
|
||||
log_debug("No active S3 mount to unmount.")
|
||||
client = Client(options)
|
||||
|
||||
def unmount_checker():
|
||||
while True:
|
||||
log_debug("Periodic check for S3 unmount...")
|
||||
time.sleep(S3_UNMOUNT_CHECK_INTERVAL)
|
||||
active_locks = redis_client.keys(f"{lock_prefix}*")
|
||||
if not active_locks:
|
||||
log_debug("No active builds detected, proceeding to unmount S3.")
|
||||
unmount_s3()
|
||||
else:
|
||||
log_debug("Active builds detected, keeping S3 mounted.")
|
||||
log_debug(f"Starting WebDAV sync from '{remote_path}' to '{local_path}'")
|
||||
client.download_sync(remote_path=remote_path, local_path=local_path)
|
||||
log_debug("WebDAV sync completed successfully.")
|
||||
|
||||
@app.route("/build", methods=["POST"])
|
||||
def build_mkdocs():
|
||||
@ -110,24 +71,22 @@ def build_mkdocs():
|
||||
log_debug(f"Build already active for website: {website}")
|
||||
return jsonify({"status": "busy", "message": f"Build already active: {website}"}), 429
|
||||
|
||||
try:
|
||||
mount_s3()
|
||||
tmp_path = f"/tmp/{website}"
|
||||
compile_path = f"{tmp_path}#compile"
|
||||
final_path = f"/srv/{website}"
|
||||
|
||||
src = f"{MINIO_MOUNT_PATH}/files/sites/@{website}/mkdocs.yml"
|
||||
tmp = f"/srv/{website}"
|
||||
try:
|
||||
sync_from_nextcloud(website, tmp_path)
|
||||
|
||||
src = os.path.join(tmp_path, "mkdocs.yml")
|
||||
|
||||
log_debug(f"Checking if mkdocs.yml exists at {src}")
|
||||
timeout = 10
|
||||
start_time = time.time()
|
||||
while not os.path.exists(src):
|
||||
if time.time() - start_time > timeout:
|
||||
log_debug(f"Timeout: {src} not found after {timeout}s")
|
||||
return jsonify({"error": f"{src} not found after mount"}), 404
|
||||
log_debug(f"Waiting for {src} to appear...")
|
||||
time.sleep(0.5)
|
||||
if not os.path.exists(src):
|
||||
log_debug(f"{src} not found after sync")
|
||||
return jsonify({"error": f"{src} not found after sync"}), 404
|
||||
|
||||
log_debug(f"Running MkDocs build: {src} -> {tmp}")
|
||||
cmd = f"mkdocs build --quiet --no-strict --config-file {src} --site-dir {tmp}"
|
||||
log_debug(f"Running MkDocs build: {src} -> {compile_path}")
|
||||
cmd = f"mkdocs build --quiet --no-strict --config-file {src} --site-dir {compile_path}"
|
||||
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
@ -137,7 +96,10 @@ def build_mkdocs():
|
||||
requests.post(error_callback, json=json_payload, headers={"Content-Type": "application/json"})
|
||||
return jsonify({"status": "error", "message": "Build failed", "error": result.stderr}), 500
|
||||
|
||||
log_debug(f"MkDocs build successful for website: {website}")
|
||||
log_debug(f"Performing differential copy from {compile_path} to {final_path}")
|
||||
subprocess.run(f"rsync -a --delete {compile_path}/ {final_path}/", shell=True, check=True)
|
||||
|
||||
log_debug(f"MkDocs build and sync successful for website: {website}")
|
||||
return jsonify({"status": "success", "message": "Build successful"}), 200
|
||||
|
||||
finally:
|
||||
@ -146,6 +108,4 @@ def build_mkdocs():
|
||||
|
||||
if __name__ == "__main__":
|
||||
log_debug("Starting Flask server on 0.0.0.0:80")
|
||||
checker_thread = Thread(target=unmount_checker, daemon=True)
|
||||
checker_thread.start()
|
||||
app.run(host="0.0.0.0", port=80)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user