diff --git a/fastdl_sync/r2_healthcheck.py b/fastdl_sync/r2_healthcheck.py new file mode 100644 index 0000000..289b388 --- /dev/null +++ b/fastdl_sync/r2_healthcheck.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +#AI slop health check for the fastdl map files. is fastdl file is unhealthy its replaced +"""Weekly R2 health check. + +Downloads every file in REMOTE to nowhere (rclone cat -> /dev/null) and times it. +Any file that takes longer than LIMIT seconds is treated as broken: it is deleted +from R2 and uploaded again from LOCAL. +""" +import os +import subprocess +import sys +import time + +REMOTE = "r2demo:unloze-r2/css_ze/maps" +LOCAL = "/home/gameservers/fastdl/css_ze/maps" +LIMIT = 60 # seconds + + +def log(msg): + print(msg, flush=True) + + +def list_remote_files(): + result = subprocess.run( + ["rclone", "lsf", "--files-only", REMOTE], + capture_output=True, text=True, check=True, + ) + return [line for line in result.stdout.splitlines() if line] + + +def timed_download(name): + """Return seconds taken, None if it exceeded LIMIT, or raise on rclone error.""" + start = time.monotonic() + try: + subprocess.run( + ["rclone", "cat", f"{REMOTE}/{name}"], + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + timeout=LIMIT, check=True, + ) + except subprocess.TimeoutExpired: + return None + return time.monotonic() - start + + +def reupload(name): + local_path = os.path.join(LOCAL, name) + if not os.path.isfile(local_path): + log(f" !! no local copy at {local_path}, leaving R2 file untouched") + return False + subprocess.run(["rclone", "deletefile", f"{REMOTE}/{name}"], check=True) + for attempt in range(1, 4): + result = subprocess.run(["rclone", "copyto", local_path, f"{REMOTE}/{name}"]) + if result.returncode == 0: + log(" re-uploaded") + return True + log(f" !! upload attempt {attempt} failed") + time.sleep(10) + log(f" !! FILE IS NOW MISSING FROM R2: {name}") + return False + + +def main(): + files = list_remote_files() + log(f"Checking {len(files)} files in {REMOTE} (limit {LIMIT}s)") + problems = 0 + + for name in files: + try: + seconds = timed_download(name) + except subprocess.CalledProcessError as e: + # A download error (e.g. network) is not proof the file is broken, + # so log it and move on rather than deleting anything. + log(f"ERROR {name}: rclone exited {e.returncode}: {e.stderr.decode().strip()}") + problems += 1 + continue + + if seconds is not None: + log(f"ok {name} ({seconds:.1f}s)") + continue + + log(f"SLOW {name} (>{LIMIT}s), replacing") + try: + if not reupload(name): + problems += 1 + except subprocess.CalledProcessError as e: + log(f" !! rclone failed while replacing: exit {e.returncode}") + problems += 1 + + log(f"Done, {problems} problem(s)") + sys.exit(1 if problems else 0) + + +if __name__ == "__main__": + main()