just a very simple health check for the bucket running once a week to confirm that files are alive and working

fastdl_sync/r2_healthcheck.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+#AI slop health check for the fastdl map files. is fastdl file is unhealthy its replaced
+"""Weekly R2 health check.
+
+Downloads every file in REMOTE to nowhere (rclone cat -> /dev/null) and times it.
+Any file that takes longer than LIMIT seconds is treated as broken: it is deleted
+from R2 and uploaded again from LOCAL.
+"""
+import os
+import subprocess
+import sys
+import time
+
+REMOTE = "r2demo:unloze-r2/css_ze/maps"
+LOCAL = "/home/gameservers/fastdl/css_ze/maps"
+LIMIT = 60  # seconds
+
+
+def log(msg):
+    print(msg, flush=True)
+
+
+def list_remote_files():
+    result = subprocess.run(
+        ["rclone", "lsf", "--files-only", REMOTE],
+        capture_output=True, text=True, check=True,
+    )
+    return [line for line in result.stdout.splitlines() if line]
+
+
+def timed_download(name):
+    """Return seconds taken, None if it exceeded LIMIT, or raise on rclone error."""
+    start = time.monotonic()
+    try:
+        subprocess.run(
+            ["rclone", "cat", f"{REMOTE}/{name}"],
+            stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
+            timeout=LIMIT, check=True,
+        )
+    except subprocess.TimeoutExpired:
+        return None
+    return time.monotonic() - start
+
+
+def reupload(name):
+    local_path = os.path.join(LOCAL, name)
+    if not os.path.isfile(local_path):
+        log(f"  !! no local copy at {local_path}, leaving R2 file untouched")
+        return False
+    subprocess.run(["rclone", "deletefile", f"{REMOTE}/{name}"], check=True)
+    for attempt in range(1, 4):
+        result = subprocess.run(["rclone", "copyto", local_path, f"{REMOTE}/{name}"])
+        if result.returncode == 0:
+            log("  re-uploaded")
+            return True
+        log(f"  !! upload attempt {attempt} failed")
+        time.sleep(10)
+    log(f"  !! FILE IS NOW MISSING FROM R2: {name}")
+    return False
+
+
+def main():
+    files = list_remote_files()
+    log(f"Checking {len(files)} files in {REMOTE} (limit {LIMIT}s)")
+    problems = 0
+
+    for name in files:
+        try:
+            seconds = timed_download(name)
+        except subprocess.CalledProcessError as e:
+            # A download error (e.g. network) is not proof the file is broken,
+            # so log it and move on rather than deleting anything.
+            log(f"ERROR {name}: rclone exited {e.returncode}: {e.stderr.decode().strip()}")
+            problems += 1
+            continue
+
+        if seconds is not None:
+            log(f"ok    {name} ({seconds:.1f}s)")
+            continue
+
+        log(f"SLOW  {name} (>{LIMIT}s), replacing")
+        try:
+            if not reupload(name):
+                problems += 1
+        except subprocess.CalledProcessError as e:
+            log(f"  !! rclone failed while replacing: exit {e.returncode}")
+            problems += 1
+
+    log(f"Done, {problems} problem(s)")
+    sys.exit(1 if problems else 0)
+
+
+if __name__ == "__main__":
+    main()