From 2385004da4bfb45d41e56f392f7ca2b7d4f5609c Mon Sep 17 00:00:00 2001 From: jenz Date: Wed, 10 May 2023 22:38:52 +0200 Subject: [PATCH] updated backups system --- file_mover/config_backups.json | 202 +++++++++++++++++++++++++-------- file_mover/file_mover.py | 14 +++ file_mover/remote_local_dir.py | 1 + file_mover/remote_sftp.py | 3 +- 4 files changed, 173 insertions(+), 47 deletions(-) diff --git a/file_mover/config_backups.json b/file_mover/config_backups.json index d9d42bd4..23c4067f 100755 --- a/file_mover/config_backups.json +++ b/file_mover/config_backups.json @@ -15,9 +15,14 @@ "path": "/etc/", "remote_type": "local_dir" }, + "local_dir_mysqldump":{ + "description": "local machine mysqldump", + "path": "", + "remote_type": "local_dir" + }, "sftp_vm_hlstats_src_home":{ "description": "sftp server for hlstats", - "hostname": "51.15.159.31", + "hostname": "5.104.107.39", "username": "nonroot", "port": "22", "path": "/home/", @@ -25,7 +30,7 @@ }, "sftp_vm_hlstats_src_etc":{ "description": "sftp server for hlstats", - "hostname": "51.15.159.31", + "hostname": "5.104.107.39", "username": "nonroot", "port": "22", "path": "/etc/", @@ -33,7 +38,7 @@ }, "sftp_vm_hlstats_src_var":{ "description": "sftp server for hlstats", - "hostname": "51.15.159.31", + "hostname": "5.104.107.39", "username": "nonroot", "port": "22", "path": "/var/", @@ -41,7 +46,7 @@ }, "sftp_vm_hlstats_src_mysqldump":{ "description": "sftp server for hlstats", - "hostname": "51.15.159.31", + "hostname": "5.104.107.39", "username": "nonroot", "port": "22", "path": "", @@ -49,7 +54,7 @@ }, "sftp_vm_xenforo_src_home":{ "description": "sftp server for xenforo", - "hostname": "163.172.225.175", + "hostname": "5.104.107.130", "username": "nonroot", "port": "22", "path": "/home/", @@ -57,7 +62,7 @@ }, "sftp_vm_xenforo_src_etc":{ "description": "sftp server for xenforo", - "hostname": "163.172.225.175", + "hostname": "5.104.107.130", "username": "nonroot", "port": "22", "path": "/etc/", @@ -65,7 +70,7 @@ }, "sftp_vm_xenforo_src_var":{ "description": "sftp server for xenforo", - "hostname": "163.172.225.175", + "hostname": "5.104.107.130", "username": "nonroot", "port": "22", "path": "/var/", @@ -73,7 +78,7 @@ }, "sftp_vm_xenforo_src_mysqldump":{ "description": "sftp server for xenforo", - "hostname": "163.172.225.175", + "hostname": "5.104.107.130", "username": "nonroot", "port": "22", "path": "", @@ -81,7 +86,7 @@ }, "sftp_vm_webservices_src_home":{ "description": "sftp server for webservices", - "hostname": "163.172.117.46", + "hostname": "5.199.136.22", "username": "nonroot", "port": "22", "path": "/home/", @@ -89,7 +94,7 @@ }, "sftp_vm_webservices_src_etc":{ "description": "sftp server for webservices", - "hostname": "163.172.117.46", + "hostname": "5.199.136.22", "username": "nonroot", "port": "22", "path": "/etc/", @@ -97,7 +102,7 @@ }, "sftp_vm_webservices_src_var":{ "description": "sftp server for webservices", - "hostname": "163.172.117.46", + "hostname": "5.199.136.22", "username": "nonroot", "port": "22", "path": "/var/", @@ -105,67 +110,123 @@ }, "sftp_vm_webservices_src_mysqldump":{ "description": "sftp server for webservices", - "hostname": "163.172.117.46", + "hostname": "5.199.136.22", "username": "nonroot", "port": "22", "path": "", "remote_type": "sftp" }, - "sftp_vm_backups_src_etc":{ - "description": "sftp server for backups/fastdl/demos", - "hostname": "163.172.119.53", + "sftp_vm_discord_autism_bot_src_etc":{ + "description": "sftp server for autismbot Discord", + "hostname": "5.199.136.34", "username": "nonroot", "port": "22", "path": "/etc/", "remote_type": "sftp" }, - "sftp_vm_backups_src_var":{ - "description": "sftp server for backups/fastdl/demos", - "hostname": "163.172.119.53", + "sftp_vm_discord_autism_bot_src_var":{ + "description": "sftp server for autismbot Discord", + "hostname": "5.199.136.34", "username": "nonroot", "port": "22", "path": "/var/", "remote_type": "sftp" }, + "sftp_vm_discord_autism_bot_src_mysqldump":{ + "description": "sftp server for autismbot Discord", + "hostname": "5.199.136.34", + "username": "nonroot", + "port": "22", + "path": "", + "remote_type": "sftp" + }, + "sftp_vm_discord_autism_bot_src_autism_bot":{ + "description": "sftp server for autismbot Discord", + "hostname": "5.199.136.34", + "username": "nonroot", + "port": "22", + "path": "/home/nonroot/autism_bot/", + "remote_type": "sftp" + }, "sftp_vm_backups_dest_ovh":{ "description": "sftp server for backups, fastdl & demos", - "hostname": "163.172.119.53", + "hostname": "5.199.136.34", "username": "nonroot", "port": "22", "path": "/home/nonroot/backups/ovh/", "remote_type": "sftp" }, - "sftp_vm_backups_dest_fastdl":{ - "description": "sftp server for backups, fastdl & demos", - "hostname": "163.172.119.53", + "sftp_vm_backups2_dest_ovh":{ + "description": "Second sftp server for backups, fastdl & demos", + "hostname": "78.31.67.70", + "username": "autismbot5", + "port": "22", + "path": "/home/autismbot5/compressed_backups/ovh/", + "remote_type": "sftp" + }, + "sftp_vm_backups_dest_discord_autism_bot":{ + "description": "sftp server for discord autism bot", + "hostname": "5.199.136.34", "username": "nonroot", "port": "22", - "path": "/home/nonroot/backups/fastdl/", + "path": "/home/nonroot/backups/autism_bot/", + "remote_type": "sftp" + }, + "sftp_vm_backups2_dest_discord_autism_bot":{ + "description": "Second sftp server for discord autism bot", + "hostname": "78.31.67.70", + "username": "autismbot5", + "port": "22", + "path": "/home/autismbot5/compressed_backups/autism_bot/", "remote_type": "sftp" }, "sftp_vm_backups_dest_hlstats":{ "description": "sftp server for backups, fastdl & demos", - "hostname": "163.172.119.53", + "hostname": "5.199.136.34", "username": "nonroot", "port": "22", "path": "/home/nonroot/backups/hlstats/", "remote_type": "sftp" }, + "sftp_vm_backups2_dest_hlstats":{ + "description": "Second sftp server for backups, fastdl & demos", + "hostname": "78.31.67.70", + "username": "autismbot5", + "port": "22", + "path": "/home/autismbot5/compressed_backups/hlstats/", + "remote_type": "sftp" + }, "sftp_vm_backups_dest_xenforo":{ "description": "sftp server for backups, fastdl & demos", - "hostname": "163.172.119.53", + "hostname": "5.199.136.34", "username": "nonroot", "port": "22", "path": "/home/nonroot/backups/xenforo/", "remote_type": "sftp" }, + "sftp_vm_backups2_dest_xenforo":{ + "description": "Second sftp server for backups, fastdl & demos", + "hostname": "78.31.67.70", + "username": "autismbot5", + "port": "22", + "path": "/home/autismbot5/compressed_backups/xenforo/", + "remote_type": "sftp" + }, "sftp_vm_backups_dest_webservices":{ "description": "sftp server for backups, fastdl & demos", - "hostname": "163.172.119.53", + "hostname": "5.199.136.34", "username": "nonroot", "port": "22", "path": "/home/nonroot/backups/webservices/", "remote_type": "sftp" + }, + "sftp_vm_backups2_dest_webservices":{ + "description": "Second sftp server for backups, fastdl & demos", + "hostname": "78.31.67.70", + "username": "autismbot5", + "port": "22", + "path": "/home/autismbot5/compressed_backups/webservices/", + "remote_type": "sftp" } }, "jobs":[ @@ -174,6 +235,7 @@ "job_description": "zips the remote webservices etc folder and moves it to backup/fastdl/demo vm", "src": "sftp_vm_webservices_src_etc", "dest": "sftp_vm_backups_dest_webservices", + "dest2": "sftp_vm_backups2_dest_webservices", "zipname":"webservices_etc_backup", "download_dir": "/home/file_mover/" }, @@ -182,6 +244,7 @@ "job_description": "zips the remote webservices var folder and moves it to backup/fastdl/demo vm", "src": "sftp_vm_webservices_src_var", "dest": "sftp_vm_backups_dest_webservices", + "dest2": "sftp_vm_backups2_dest_webservices", "zipname":"webservices_var_backup", "download_dir": "/home/file_mover/" }, @@ -190,6 +253,7 @@ "job_description": "zips the remote webservices home folder and moves it to backup/fastdl/demo vm", "src": "sftp_vm_webservices_src_home", "dest": "sftp_vm_backups_dest_webservices", + "dest2": "sftp_vm_backups2_dest_webservices", "zipname":"webservices_home_backup", "download_dir": "/home/file_mover/" }, @@ -198,6 +262,7 @@ "job_description": "dumps the remote webservices vm databases and moves them to backup/fastdl/demo vm", "src": "sftp_vm_webservices_src_mysqldump", "dest": "sftp_vm_backups_dest_webservices", + "dest2": "sftp_vm_backups2_dest_webservices", "zipname":"webservices_mysqldump_backup", "download_dir": "/home/file_mover/" }, @@ -206,6 +271,7 @@ "job_description": "zips the remote xenforo etc folder and moves it to backup/fastdl/demo vm", "src": "sftp_vm_xenforo_src_etc", "dest": "sftp_vm_backups_dest_xenforo", + "dest2": "sftp_vm_backups2_dest_xenforo", "zipname":"xenforo_etc_backup", "download_dir": "/home/file_mover/" }, @@ -214,6 +280,7 @@ "job_description": "zips the remote xenforo var folder and moves it to backup/fastdl/demo vm", "src": "sftp_vm_xenforo_src_var", "dest": "sftp_vm_backups_dest_xenforo", + "dest2": "sftp_vm_backups2_dest_xenforo", "zipname":"xenforo_var_backup", "download_dir": "/home/file_mover/" }, @@ -222,6 +289,7 @@ "job_description": "zips the remote xenforo home folder and moves it to backup/fastdl/demo vm", "src": "sftp_vm_xenforo_src_home", "dest": "sftp_vm_backups_dest_xenforo", + "dest2": "sftp_vm_backups2_dest_xenforo", "zipname":"xenforo_home_backup", "download_dir": "/home/file_mover/" }, @@ -230,6 +298,7 @@ "job_description": "dumps the remote xenforo vm databases and moves them to backup/fastdl/demo vm", "src": "sftp_vm_xenforo_src_mysqldump", "dest": "sftp_vm_backups_dest_xenforo", + "dest2": "sftp_vm_backups2_dest_xenforo", "zipname":"xenforo_mysqldump_backup", "download_dir": "/home/file_mover/" }, @@ -238,6 +307,7 @@ "job_description": "zips the remote hlstats etc folder and moves it to backup/fastdl/demo vm", "src": "sftp_vm_hlstats_src_etc", "dest": "sftp_vm_backups_dest_hlstats", + "dest2": "sftp_vm_backups2_dest_hlstats", "zipname":"hlstats_etc_backup", "download_dir": "/home/file_mover/" }, @@ -246,6 +316,7 @@ "job_description": "zips the remote hlstats var folder and moves it to backup/fastdl/demo vm", "src": "sftp_vm_hlstats_src_var", "dest": "sftp_vm_backups_dest_hlstats", + "dest2": "sftp_vm_backups2_dest_hlstats", "zipname":"hlstats_var_backup", "download_dir": "/home/file_mover/" }, @@ -254,6 +325,7 @@ "job_description": "dumps the remote hlstats vm databases and moves them to backup/fastdl/demo vm", "src": "sftp_vm_hlstats_src_mysqldump", "dest": "sftp_vm_backups_dest_hlstats", + "dest2": "sftp_vm_backups2_dest_hlstats", "zipname":"hlstats_mysqldump_backup", "download_dir": "/home/file_mover/" }, @@ -262,23 +334,44 @@ "job_description": "zips the remote hlstats home folder and moves it to backup/fastdl/demo vm", "src": "sftp_vm_hlstats_src_home", "dest": "sftp_vm_backups_dest_hlstats", + "dest2": "sftp_vm_backups2_dest_hlstats", "zipname":"hlstats_home_backup", "download_dir": "/home/file_mover/" }, { - "job_name": "backup_remote_fastdl_etc", - "job_description": "zips the remote backups/fastdl/demos etc folder and moves it to backup/fastdl/demo vm", - "src": "sftp_vm_backups_src_etc", - "dest": "sftp_vm_backups_dest_fastdl", - "zipname":"fastdl_etc_backup", + "job_name": "backup_remote_autism_bot_etc", + "job_description": "zips the remote discord autism bot etc folder and moves it to discord autism bot vm", + "src": "sftp_vm_discord_autism_bot_src_etc", + "dest": "sftp_vm_backups_dest_discord_autism_bot", + "dest2": "sftp_vm_backups2_dest_discord_autism_bot", + "zipname":"discord_autism_bot_etc_backup", "download_dir": "/home/file_mover/" }, - { - "job_name": "backup_remote_fastdl_var", - "job_description": "zips the remote backups/fastdl/demos var folder and moves it to backup/fastdl/demo vm", - "src": "sftp_vm_backups_src_var", - "dest": "sftp_vm_backups_dest_fastdl", - "zipname":"fastdl_var_backup", + { + "job_name": "backup_remote_autism_bot_var", + "job_description": "zips the remote discord autism bot var folder and moves it to discord autism bot vm", + "src": "sftp_vm_discord_autism_bot_src_var", + "dest": "sftp_vm_backups_dest_discord_autism_bot", + "dest2": "sftp_vm_backups2_dest_discord_autism_bot", + "zipname":"discord_autism_bot_var_backup", + "download_dir": "/home/file_mover/" + }, + { + "job_name": "backup_remote_autism_bot_mysqldump", + "job_description": "dumps the remote discord autism bot vm databases and moves them to discord autism bot vm", + "src": "sftp_vm_discord_autism_bot_src_mysqldump", + "dest": "sftp_vm_backups_dest_discord_autism_bot", + "dest2": "sftp_vm_backups2_dest_discord_autism_bot", + "zipname":"discord_autism_bot_mysqldump_backup", + "download_dir": "/home/file_mover/" + }, + { + "job_name": "backup_remote_autism_bot_discord", + "job_description": "zips the remote autism bot discord autism bot discord folder and moves it to backup/demo vm", + "src": "sftp_vm_discord_autism_bot_src_autism_bot", + "dest": "sftp_vm_backups_dest_discord_autism_bot", + "dest2": "sftp_vm_backups2_dest_discord_autism_bot", + "zipname":"discord_autism_bot_folder_backup", "download_dir": "/home/file_mover/" }, { @@ -286,6 +379,7 @@ "job_description": "zips the local gameservers folder and moves it to backup/fastdl/demo vm", "src": "local_dir_gameservers", "dest": "sftp_vm_backups_dest_ovh", + "dest2": "sftp_vm_backups2_dest_ovh", "zipname":"ovh_gameservers_backup" }, { @@ -293,6 +387,7 @@ "job_description": "zips the local var folder and moves it to backup/fastdl/demo vm", "src": "local_dir_var", "dest": "sftp_vm_backups_dest_ovh", + "dest2": "sftp_vm_backups2_dest_ovh", "zipname":"ovh_var_backup" }, { @@ -300,33 +395,48 @@ "job_description": "zips the local etc folder and moves it to backup/fastdl/demo vm", "src": "local_dir_etc", "dest": "sftp_vm_backups_dest_ovh", + "dest2": "sftp_vm_backups2_dest_ovh", "zipname":"ovh_etc_backup" + }, + { + "job_name": "backup_local_mysqldump", + "job_description": "zips the local databases and moves it to backup/fastdl/demo vm", + "src": "local_dir_mysqldump", + "dest": "sftp_vm_backups_dest_ovh", + "dest2": "sftp_vm_backups2_dest_ovh", + "zipname":"ovh_mysqldump_backup" } ], "settings": { "sftp": { "remote_attempts": "5", "remote_delay": "15", - "ip":{ + "5.199.136.34":{ "username": { - "password": "password" + "password": "" } }, - "ip":{ + "5.199.136.22":{ "username": { - "password": "password" + "password": "" } }, - "ip":{ + "5.104.107.130":{ "username": { - "password": "password" + "password": "" } }, - "ip":{ + "5.104.107.39":{ "username": { - "password": "password" + "password": "" } - } + }, + "78.31.67.70": { + "username": { + "password": "" + } + } } } } + diff --git a/file_mover/file_mover.py b/file_mover/file_mover.py index bc09b350..142d723e 100755 --- a/file_mover/file_mover.py +++ b/file_mover/file_mover.py @@ -53,9 +53,11 @@ def main(): for index, job in enumerate(jobs): src = create_remote(remotes[job["src"]], settings) dest = create_remote(remotes[job["dest"]], settings) + dest2 = create_remote(remotes[job["dest2"]], settings) if index == 0: #delete old files (not very adaptable, cant handle dirs in dirs) dest.delete_remote_zips("/home/nonroot/backups") + dest2.delete_remote_zips("/home/autismbot5/compressed_backups") if 'demo_' in job["job_name"]: source_files = src.list_dir() distribute_files(source_files, dest) @@ -68,6 +70,10 @@ def main(): log_msg = ''.join(["failed putting file: ", str(pathfile)]) logging.warning(log_msg) sys.exit(1) + if not dest2.put(zip_file_path, dest2.path): + log_msg = ''.join(["failed putting file: ", str(pathfile)]) + logging.warning(log_msg) + sys.exit(1) src.delete_local_zip(zip_file_path) elif "mysqldump" in job["job_name"]: src.remote_channel_command(job, "mysqldump") @@ -85,6 +91,10 @@ def main(): logging.warning(f'failed putting local zip {local_zip_path_name} at job {job}') src.delete_local_zip(local_zip_path_name) sys.exit(1) + if not dest2.put(local_zip_path_name, dest2.path): + logging.warning(f'failed putting local zip {local_zip_path_name} at job {job}') + src.delete_local_zip(local_zip_path_name) + sys.exit(1) src.delete_local_zip(local_zip_path_name) elif "backup_remote" in job["job_name"]: zipname = src.remote_channel_command(job, "tar_recursive") @@ -101,6 +111,10 @@ def main(): logging.warning(f'failed putting local zip {local_zip_path_name} at job {job}') src.delete_local_zip(local_zip_path_name) sys.exit(1) + if not dest2.put(local_zip_path_name, dest2.path): + logging.warning(f'failed putting local zip {local_zip_path_name} at job {job}') + src.delete_local_zip(local_zip_path_name) + sys.exit(1) src.delete_local_zip(local_zip_path_name) logging.info(('finished job: ', job)) diff --git a/file_mover/remote_local_dir.py b/file_mover/remote_local_dir.py index 237d6daf..403baefe 100755 --- a/file_mover/remote_local_dir.py +++ b/file_mover/remote_local_dir.py @@ -53,3 +53,4 @@ class local_dir_remote: if (str(zip_file_path).endswith('.tar.xz')): os.remove(zip_file_path) print('removed tar xz: ', zip_file_path) + diff --git a/file_mover/remote_sftp.py b/file_mover/remote_sftp.py index 5f4e6535..5e1af7bc 100755 --- a/file_mover/remote_sftp.py +++ b/file_mover/remote_sftp.py @@ -133,7 +133,7 @@ class sftp_remote: local_temp_folder = local_path_get + "/tempfolder/" try: self.connect() - #print('remote_path chdir: ', remote_path) + print('remote_path chdir: ', remote_path) self.sftp.chdir(remote_path) # Test if remote_path exists except IOError: self.sftp.mkdir(remote_path) @@ -204,3 +204,4 @@ class sftp_remote: def delete_local_zip(self, local_zip_path_name): os.remove(local_zip_path_name) +