95 lines
2.8 KiB
Python
95 lines
2.8 KiB
Python
#!/usr/bin/env python3
|
|
#AI slop health check for the fastdl map files. is fastdl file is unhealthy its replaced
|
|
"""Weekly R2 health check.
|
|
|
|
Downloads every file in REMOTE to nowhere (rclone cat -> /dev/null) and times it.
|
|
Any file that takes longer than LIMIT seconds is treated as broken: it is deleted
|
|
from R2 and uploaded again from LOCAL.
|
|
"""
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
REMOTE = "r2demo:unloze-r2/css_ze/maps"
|
|
LOCAL = "/home/gameservers/fastdl/css_ze/maps"
|
|
LIMIT = 60 # seconds
|
|
|
|
|
|
def log(msg):
|
|
print(msg, flush=True)
|
|
|
|
|
|
def list_remote_files():
|
|
result = subprocess.run(
|
|
["rclone", "lsf", "--files-only", REMOTE],
|
|
capture_output=True, text=True, check=True,
|
|
)
|
|
return [line for line in result.stdout.splitlines() if line]
|
|
|
|
|
|
def timed_download(name):
|
|
"""Return seconds taken, None if it exceeded LIMIT, or raise on rclone error."""
|
|
start = time.monotonic()
|
|
try:
|
|
subprocess.run(
|
|
["rclone", "cat", f"{REMOTE}/{name}"],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
|
|
timeout=LIMIT, check=True,
|
|
)
|
|
except subprocess.TimeoutExpired:
|
|
return None
|
|
return time.monotonic() - start
|
|
|
|
|
|
def reupload(name):
|
|
local_path = os.path.join(LOCAL, name)
|
|
if not os.path.isfile(local_path):
|
|
log(f" !! no local copy at {local_path}, leaving R2 file untouched")
|
|
return False
|
|
subprocess.run(["rclone", "deletefile", f"{REMOTE}/{name}"], check=True)
|
|
for attempt in range(1, 4):
|
|
result = subprocess.run(["rclone", "copyto", local_path, f"{REMOTE}/{name}"])
|
|
if result.returncode == 0:
|
|
log(" re-uploaded")
|
|
return True
|
|
log(f" !! upload attempt {attempt} failed")
|
|
time.sleep(10)
|
|
log(f" !! FILE IS NOW MISSING FROM R2: {name}")
|
|
return False
|
|
|
|
|
|
def main():
|
|
files = list_remote_files()
|
|
log(f"Checking {len(files)} files in {REMOTE} (limit {LIMIT}s)")
|
|
problems = 0
|
|
|
|
for name in files:
|
|
try:
|
|
seconds = timed_download(name)
|
|
except subprocess.CalledProcessError as e:
|
|
# A download error (e.g. network) is not proof the file is broken,
|
|
# so log it and move on rather than deleting anything.
|
|
log(f"ERROR {name}: rclone exited {e.returncode}: {e.stderr.decode().strip()}")
|
|
problems += 1
|
|
continue
|
|
|
|
if seconds is not None:
|
|
log(f"ok {name} ({seconds:.1f}s)")
|
|
continue
|
|
|
|
log(f"SLOW {name} (>{LIMIT}s), replacing")
|
|
try:
|
|
if not reupload(name):
|
|
problems += 1
|
|
except subprocess.CalledProcessError as e:
|
|
log(f" !! rclone failed while replacing: exit {e.returncode}")
|
|
problems += 1
|
|
|
|
log(f"Done, {problems} problem(s)")
|
|
sys.exit(1 if problems else 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|