#!/usr/bin/env python3 #AI slop health check for the fastdl map files. is fastdl file is unhealthy its replaced """Weekly R2 health check. Downloads every file in REMOTE to nowhere (rclone cat -> /dev/null) and times it. Any file that takes longer than LIMIT seconds is treated as broken: it is deleted from R2 and uploaded again from LOCAL. """ import os import subprocess import sys import time REMOTE = "r2demo:unloze-r2/css_ze/maps" LOCAL = "/home/gameservers/fastdl/css_ze/maps" LIMIT = 60 # seconds def log(msg): print(msg, flush=True) def list_remote_files(): result = subprocess.run( ["rclone", "lsf", "--files-only", REMOTE], capture_output=True, text=True, check=True, ) return [line for line in result.stdout.splitlines() if line] def timed_download(name): """Return seconds taken, None if it exceeded LIMIT, or raise on rclone error.""" start = time.monotonic() try: subprocess.run( ["rclone", "cat", f"{REMOTE}/{name}"], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=LIMIT, check=True, ) except subprocess.TimeoutExpired: return None return time.monotonic() - start def reupload(name): local_path = os.path.join(LOCAL, name) if not os.path.isfile(local_path): log(f" !! no local copy at {local_path}, leaving R2 file untouched") return False subprocess.run(["rclone", "deletefile", f"{REMOTE}/{name}"], check=True) for attempt in range(1, 4): result = subprocess.run(["rclone", "copyto", local_path, f"{REMOTE}/{name}"]) if result.returncode == 0: log(" re-uploaded") return True log(f" !! upload attempt {attempt} failed") time.sleep(10) log(f" !! FILE IS NOW MISSING FROM R2: {name}") return False def main(): files = list_remote_files() log(f"Checking {len(files)} files in {REMOTE} (limit {LIMIT}s)") problems = 0 for name in files: try: seconds = timed_download(name) except subprocess.CalledProcessError as e: # A download error (e.g. network) is not proof the file is broken, # so log it and move on rather than deleting anything. log(f"ERROR {name}: rclone exited {e.returncode}: {e.stderr.decode().strip()}") problems += 1 continue if seconds is not None: log(f"ok {name} ({seconds:.1f}s)") continue log(f"SLOW {name} (>{LIMIT}s), replacing") try: if not reupload(name): problems += 1 except subprocess.CalledProcessError as e: log(f" !! rclone failed while replacing: exit {e.returncode}") problems += 1 log(f"Done, {problems} problem(s)") sys.exit(1 if problems else 0) if __name__ == "__main__": main()