From f174a8a5b14457d60954239f6f7404d9da5f9e07 Mon Sep 17 00:00:00 2001 From: jenz Date: Sat, 26 Jul 2025 15:36:27 +0200 Subject: [PATCH] updated fastdl_sync with gmod stuff. used specific path structures that i was asked to use. --- fastdl_sync/r2_update.sh | 37 +++---- fastdl_sync/run-docker-cp.py | 109 ++++++++++++++++++--- fastdl_sync/run-fastdl.sh | 3 +- r2_upload/.config/rclone/rclone.conf | 8 -- r2_upload/shell/r2_update.sh | 45 --------- r2_upload/systemd/r2_bucket_rclone.service | 9 -- r2_upload/systemd/r2_bucket_rclone.timer | 9 -- 7 files changed, 118 insertions(+), 102 deletions(-) delete mode 100644 r2_upload/.config/rclone/rclone.conf delete mode 100755 r2_upload/shell/r2_update.sh delete mode 100644 r2_upload/systemd/r2_bucket_rclone.service delete mode 100644 r2_upload/systemd/r2_bucket_rclone.timer diff --git a/fastdl_sync/r2_update.sh b/fastdl_sync/r2_update.sh index 464ead0..55aaa78 100644 --- a/fastdl_sync/r2_update.sh +++ b/fastdl_sync/r2_update.sh @@ -1,20 +1,20 @@ -#!/usr/bin/sh -#run full reclone when arguement is given -if [ $# -eq 1 ] - then - echo 'doing long run with all files being updated.' - #ZE server - rclone copy /home/gameservers/fastdl/css_ze/materials r2demo:/css_ze/materials/ - rclone copy /home/gameservers/fastdl/css_ze/models r2demo:/css_ze/models/ - rclone copy /home/gameservers/fastdl/css_ze/maps r2demo:/css_ze/maps/ - rclone copy /home/gameservers/fastdl/css_ze/sound r2demo:/css_ze/sound/ - #ZR server - rclone copy /home/gameservers/fastdl/css_zr/materials r2demo:/css_zr/materials/ - rclone copy /home/gameservers/fastdl/css_zr/models r2demo:/css_zr/models/ - rclone copy /home/gameservers/fastdl/css_zr/maps r2demo:/css_zr/maps/ - rclone copy /home/gameservers/fastdl/css_zr/sound r2demo:/css_zr/sound/ - #MG server - rclone copy /home/gameservers/fastdl/css_mg/materials r2demo:/css_mg/materials/ +#!/usr/bin/sh +#run full reclone when arguement is given +if [ $# -eq 1 ] + then + echo 'doing long run with all files being updated.' + #ZE server + rclone copy /home/gameservers/fastdl/css_ze/materials r2demo:/css_ze/materials/ + rclone copy /home/gameservers/fastdl/css_ze/models r2demo:/css_ze/models/ + rclone copy /home/gameservers/fastdl/css_ze/maps r2demo:/css_ze/maps/ + rclone copy /home/gameservers/fastdl/css_ze/sound r2demo:/css_ze/sound/ + #ZR server + rclone copy /home/gameservers/fastdl/css_zr/materials r2demo:/css_zr/materials/ + rclone copy /home/gameservers/fastdl/css_zr/models r2demo:/css_zr/models/ + rclone copy /home/gameservers/fastdl/css_zr/maps r2demo:/css_zr/maps/ + rclone copy /home/gameservers/fastdl/css_zr/sound r2demo:/css_zr/sound/ + #MG server + rclone copy /home/gameservers/fastdl/css_mg/materials r2demo:/css_mg/materials/ rclone copy /home/gameservers/fastdl/css_mg/models r2demo:/css_mg/models/ rclone copy /home/gameservers/fastdl/css_mg/maps r2demo:/css_mg/maps/ rclone copy /home/gameservers/fastdl/css_mg/sound r2demo:/css_mg/sound/ @@ -45,5 +45,8 @@ else rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/svencoop/svencoop_addon r2demo:/svencoop/ #gmod zs rclone copy --max-age 30m --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/maps r2demo:/gmod_zs/maps/ + rclone copy --max-age 30m --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/gamemodes r2demo:/gmod_zs/gamemodes/ + rclone copy --max-age 30m --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/addons r2demo:/gmod_zs/addons/ echo 'Finished short run with files younger than 30 minutes.' fi + diff --git a/fastdl_sync/run-docker-cp.py b/fastdl_sync/run-docker-cp.py index a3187dc..0e12b9c 100644 --- a/fastdl_sync/run-docker-cp.py +++ b/fastdl_sync/run-docker-cp.py @@ -1,36 +1,119 @@ #!/usr/local/bin/python3.5 - -import subprocess - + +import subprocess +import sys + def compare_lists(docker_find, ovh_find): - d = [] + d = [] for file_docker in docker_find: file_name_docker = str(file_docker).split("/", -1)[-1].replace("'", '') found = False for file_ovh in ovh_find: file_name_ovh = str(file_ovh).split("/", -1)[-1].replace("'", '') if file_name_ovh == file_name_docker: - found = True - break + found = True + break if not found: - d.append(file_docker) + d.append(file_docker) return d -def files_to_move(files, directory): +def files_to_move(files): for file in files: file = file.decode("utf-8") - str1 = f"docker cp debian-server:{file} ../gmod_zs/fastdl/{directory}/" - #print(str1) + wanted_path = file.split("/garrysmod/")[1] + wanted_path = wanted_path.rsplit("/", 1)[0] + str1 = f"docker cp debian-server:{file} ../gmod_zs/fastdl/{wanted_path}" + #print('str1: ', str1) + ##sys.exit(1) + subprocess.Popen([f"mkdir -p ../gmod_zs/fastdl/{wanted_path}"], shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).communicate()[0] subprocess.Popen([str1], shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).communicate()[0] - if __name__ == "__main__": #find maps in docker docker_find = subprocess.Popen(["docker exec debian-server find /home/gameservers_zs/gmod_zs_1/garrysmod/maps -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() #find maps on our OVH machine ovh_find = subprocess.Popen(["find /home/gameservers/gmod_zs/fastdl/maps/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() #if maps are in docker but not OVH we need to list them - maps_to_move = compare_lists(docker_find, ovh_find) + content_to_move = compare_lists(docker_find, ovh_find) #move the files to OVH. - files_to_move(maps_to_move, "maps") + files_to_move(content_to_move) + + #find gamemodes in docker + docker_find = subprocess.Popen(["docker exec debian-server find /home/gameservers_zs/gmod_zs_1/garrysmod/gamemodes/zombiesurvival/content/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + #find gamemodes on our OVH machine + ovh_find = subprocess.Popen(["find /home/gameservers/gmod_zs/fastdl/gamemodes/zombiesurvival/content/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + #if are in docker but not OVH we need to list them + content_to_move = compare_lists(docker_find, ovh_find) + #move the files to OVH. + files_to_move(content_to_move) + + + docker_find = subprocess.Popen(["docker exec debian-server find /home/gameservers_zs/gmod_zs_1/garrysmod/addons/atlaschat/materials/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + ovh_find = subprocess.Popen(["find /home/gameservers/gmod_zs/fastdl/gamemodes/addons/atlaschat/materials/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + content_to_move = compare_lists(docker_find, ovh_find) + #move the files to OVH. + files_to_move(content_to_move) + + + docker_find = subprocess.Popen(["docker exec debian-server find /home/gameservers_zs/gmod_zs_1/garrysmod/addons/atlaschat/resource/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + ovh_find = subprocess.Popen(["find /home/gameservers/gmod_zs/fastdl/gamemodes/addons/atlaschat/resource/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + content_to_move = compare_lists(docker_find, ovh_find) + #move the files to OVH. + files_to_move(content_to_move) + + + docker_find = subprocess.Popen(["docker exec debian-server find /home/gameservers_zs/gmod_zs_1/garrysmod/addons/uzscontent/materials/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + ovh_find = subprocess.Popen(["find /home/gameservers/gmod_zs/fastdl/gamemodes/addons/uzscontent/materials/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + content_to_move = compare_lists(docker_find, ovh_find) + #move the files to OVH. + files_to_move(content_to_move) + + + docker_find = subprocess.Popen(["docker exec debian-server find /home/gameservers_zs/gmod_zs_1/garrysmod/addons/uzscontent/models/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + ovh_find = subprocess.Popen(["find /home/gameservers/gmod_zs/fastdl/gamemodes/addons/uzscontent/models/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + content_to_move = compare_lists(docker_find, ovh_find) + #move the files to OVH. + files_to_move(content_to_move) + + + docker_find = subprocess.Popen(["docker exec debian-server find /home/gameservers_zs/gmod_zs_1/garrysmod/addons/uzscontent/particles/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + ovh_find = subprocess.Popen(["find /home/gameservers/gmod_zs/fastdl/gamemodes/addons/uzscontent/particles/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + content_to_move = compare_lists(docker_find, ovh_find) + #move the files to OVH. + files_to_move(content_to_move) + + + docker_find = subprocess.Popen(["docker exec debian-server find /home/gameservers_zs/gmod_zs_1/garrysmod/addons/uzscontent/resource/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + ovh_find = subprocess.Popen(["find /home/gameservers/gmod_zs/fastdl/gamemodes/addons/uzscontent/resource/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + content_to_move = compare_lists(docker_find, ovh_find) + #move the files to OVH. + files_to_move(content_to_move) + + + docker_find = subprocess.Popen(["docker exec debian-server find /home/gameservers_zs/gmod_zs_1/garrysmod/addons/uzscontent/sound/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + ovh_find = subprocess.Popen(["find /home/gameservers/gmod_zs/fastdl/gamemodes/addons/uzscontent/sound/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + content_to_move = compare_lists(docker_find, ovh_find) + #move the files to OVH. + files_to_move(content_to_move) + + + docker_find = subprocess.Popen(["docker exec debian-server find /home/gameservers_zs/gmod_zs_1/garrysmod/addons/vipmodels/materials/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + ovh_find = subprocess.Popen(["find /home/gameservers/gmod_zs/fastdl/gamemodes/addons/vipmodels/materials/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + content_to_move = compare_lists(docker_find, ovh_find) + #move the files to OVH. + files_to_move(content_to_move) + + + docker_find = subprocess.Popen(["docker exec debian-server find /home/gameservers_zs/gmod_zs_1/garrysmod/addons/vipmodels/models/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + ovh_find = subprocess.Popen(["find /home/gameservers/gmod_zs/fastdl/gamemodes/addons/vipmodels/models/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + content_to_move = compare_lists(docker_find, ovh_find) + #move the files to OVH. + files_to_move(content_to_move) + + + docker_find = subprocess.Popen(["docker exec debian-server find /home/gameservers_zs/gmod_zs_1/garrysmod/addons/wepmodels/models/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + ovh_find = subprocess.Popen(["find /home/gameservers/gmod_zs/fastdl/gamemodes/addons/wepmodels/models/ -type f"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate()[0].splitlines() + content_to_move = compare_lists(docker_find, ovh_find) + #move the files to OVH. + files_to_move(content_to_move) diff --git a/fastdl_sync/run-fastdl.sh b/fastdl_sync/run-fastdl.sh index d669153..d1ac325 100755 --- a/fastdl_sync/run-fastdl.sh +++ b/fastdl_sync/run-fastdl.sh @@ -12,4 +12,5 @@ screen -A -m -d -S fastdl_css_ze_uk python3 fastdl_ftp_uk.py -t 2 ../css screen -A -m -d -S fastdl_css_zr_uk python3 fastdl_ftp_uk.py -t 2 ../css_zr/cstrike/{maps,materials,models,sound} ftp://127.0.0.1/fastdl/css_zr screen -A -m -d -S fastdl_css_mg_uk python3 fastdl_ftp_uk.py -t 2 ../css_mg/cstrike/{maps,materials,models,sound} ftp://127.0.0.1/fastdl/css_mg screen -A -m -d -S fastdl_svencoop_uk python3 fastdl_ftp_uk_sven.py -t 2 ../svencoop/svencoop_addon/ ftp://127.0.0.1/fastdl/svencoop -screen -A -m -d -S fastdl_gmod_zs_uk python3 fastdl_ftp_uk.py -t 2 ../gmod_zs/fastdl/{maps,materials,models,sound} ftp://127.0.0.1/fastdl/gmod_zs +screen -A -m -d -S fastdl_gmod_zs_uk python3 fastdl_ftp_uk.py -t 2 ../gmod_zs/fastdl/{maps,gamemodes,addons,materials,models,sound} ftp://127.0.0.1/fastdl/gmod_zs + diff --git a/r2_upload/.config/rclone/rclone.conf b/r2_upload/.config/rclone/rclone.conf deleted file mode 100644 index a45343f..0000000 --- a/r2_upload/.config/rclone/rclone.conf +++ /dev/null @@ -1,8 +0,0 @@ -[r2demo] -type = s3 -provider = Cloudflare -access_key_id = -secret_access_key = -endpoint = -acl = private - diff --git a/r2_upload/shell/r2_update.sh b/r2_upload/shell/r2_update.sh deleted file mode 100755 index 2f7fdfb..0000000 --- a/r2_upload/shell/r2_update.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/sh -#run full reclone when arguement is given -if [ $# -eq 1 ] - then - echo 'doing long run with all files being updated.' - #ZE server - rclone copy /home/gameservers/fastdl/css_ze/materials r2demo:/css_ze/materials/ - rclone copy /home/gameservers/fastdl/css_ze/models r2demo:/css_ze/models/ - rclone copy /home/gameservers/fastdl/css_ze/maps r2demo:/css_ze/maps/ - rclone copy /home/gameservers/fastdl/css_ze/sound r2demo:/css_ze/sound/ - #ZR server - rclone copy /home/gameservers/fastdl/css_zr/materials r2demo:/css_zr/materials/ - rclone copy /home/gameservers/fastdl/css_zr/models r2demo:/css_zr/models/ - rclone copy /home/gameservers/fastdl/css_zr/maps r2demo:/css_zr/maps/ - rclone copy /home/gameservers/fastdl/css_zr/sound r2demo:/css_zr/sound/ - #MG server - rclone copy /home/gameservers/fastdl/css_mg/materials r2demo:/css_mg/materials/ - rclone copy /home/gameservers/fastdl/css_mg/models r2demo:/css_mg/models/ - rclone copy /home/gameservers/fastdl/css_mg/maps r2demo:/css_mg/maps/ - rclone copy /home/gameservers/fastdl/css_mg/sound r2demo:/css_mg/sound/ - #svencoop - rclone copy /home/gameservers/fastdl/svencoop/svencoop_addon r2demo:/svencoop/ - echo 'Finished long run with all files updated.' -#else if no arguement given run default with 30minute maximum. -else - echo 'doing short run with files younger than 30 minutes.' - #ZE server - rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_ze/materials r2demo:/css_ze/materials/ - rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_ze/models r2demo:/css_ze/models/ - rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_ze/maps r2demo:/css_ze/maps/ - rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_ze/sound r2demo:/css_ze/sound/ - #ZR server - rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_zr/materials r2demo:/css_zr/materials/ - rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_zr/models r2demo:/css_zr/models/ - rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_zr/maps r2demo:/css_zr/maps/ - rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_zr/sound r2demo:/css_zr/sound/ - #MG server - rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_mg/materials r2demo:/css_mg/materials/ - rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_mg/models r2demo:/css_mg/models/ - rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_mg/maps r2demo:/css_mg/maps/ - rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_mg/sound r2demo:/css_mg/sound/ - #svencoop - rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/svencoop/svencoop_addon r2demo:/svencoop/ - echo 'Finished short run with files younger than 30 minutes.' -fi diff --git a/r2_upload/systemd/r2_bucket_rclone.service b/r2_upload/systemd/r2_bucket_rclone.service deleted file mode 100644 index 70983b5..0000000 --- a/r2_upload/systemd/r2_bucket_rclone.service +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Rclones to R2 bucket every 10 minutes - -[Service] -Type=simple -User=gameservers -Group=gameservers -WorkingDirectory=/home/gameservers/fastdl_sync -ExecStart=/home/gameservers/fastdl_sync/r2_update.sh diff --git a/r2_upload/systemd/r2_bucket_rclone.timer b/r2_upload/systemd/r2_bucket_rclone.timer deleted file mode 100644 index 0b89e56..0000000 --- a/r2_upload/systemd/r2_bucket_rclone.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=rclones new content every 10 minutes -Requires=r2_bucket_rclone.service - -[Timer] -OnCalendar=*-*-* *:0,10,20,30,40,50 - -[Install] -WantedBy=multi-user.target