diff --git a/file_mover/config_backups.json b/file_mover/config_backups.json index 23c4067f..714bce96 100755 --- a/file_mover/config_backups.json +++ b/file_mover/config_backups.json @@ -5,434 +5,78 @@ "path": "/home/gameservers/", "remote_type": "local_dir" }, - "local_dir_var": { - "description": "local machine var", - "path": "/var/", - "remote_type": "local_dir" - }, - "local_dir_etc": { - "description": "local machine etc", - "path": "/etc/", - "remote_type": "local_dir" - }, - "local_dir_mysqldump":{ - "description": "local machine mysqldump", - "path": "", - "remote_type": "local_dir" - }, - "sftp_vm_hlstats_src_home":{ - "description": "sftp server for hlstats", - "hostname": "5.104.107.39", - "username": "nonroot", - "port": "22", - "path": "/home/", - "remote_type": "sftp" - }, - "sftp_vm_hlstats_src_etc":{ - "description": "sftp server for hlstats", - "hostname": "5.104.107.39", - "username": "nonroot", - "port": "22", - "path": "/etc/", - "remote_type": "sftp" - }, - "sftp_vm_hlstats_src_var":{ - "description": "sftp server for hlstats", - "hostname": "5.104.107.39", - "username": "nonroot", - "port": "22", - "path": "/var/", - "remote_type": "sftp" - }, - "sftp_vm_hlstats_src_mysqldump":{ - "description": "sftp server for hlstats", - "hostname": "5.104.107.39", - "username": "nonroot", - "port": "22", - "path": "", - "remote_type": "sftp" - }, - "sftp_vm_xenforo_src_home":{ - "description": "sftp server for xenforo", - "hostname": "5.104.107.130", - "username": "nonroot", - "port": "22", - "path": "/home/", - "remote_type": "sftp" - }, - "sftp_vm_xenforo_src_etc":{ - "description": "sftp server for xenforo", - "hostname": "5.104.107.130", - "username": "nonroot", - "port": "22", - "path": "/etc/", - "remote_type": "sftp" - }, - "sftp_vm_xenforo_src_var":{ - "description": "sftp server for xenforo", - "hostname": "5.104.107.130", - "username": "nonroot", - "port": "22", - "path": "/var/", - "remote_type": "sftp" - }, - "sftp_vm_xenforo_src_mysqldump":{ - "description": "sftp server for xenforo", - "hostname": "5.104.107.130", - "username": "nonroot", - "port": "22", - "path": "", - "remote_type": "sftp" - }, - "sftp_vm_webservices_src_home":{ - "description": "sftp server for webservices", - "hostname": "5.199.136.22", - "username": "nonroot", - "port": "22", - "path": "/home/", - "remote_type": "sftp" - }, - "sftp_vm_webservices_src_etc":{ - "description": "sftp server for webservices", - "hostname": "5.199.136.22", - "username": "nonroot", - "port": "22", - "path": "/etc/", - "remote_type": "sftp" - }, - "sftp_vm_webservices_src_var":{ - "description": "sftp server for webservices", - "hostname": "5.199.136.22", - "username": "nonroot", - "port": "22", - "path": "/var/", - "remote_type": "sftp" - }, - "sftp_vm_webservices_src_mysqldump":{ - "description": "sftp server for webservices", - "hostname": "5.199.136.22", - "username": "nonroot", - "port": "22", - "path": "", - "remote_type": "sftp" - }, - "sftp_vm_discord_autism_bot_src_etc":{ - "description": "sftp server for autismbot Discord", - "hostname": "5.199.136.34", - "username": "nonroot", - "port": "22", - "path": "/etc/", - "remote_type": "sftp" - }, - "sftp_vm_discord_autism_bot_src_var":{ - "description": "sftp server for autismbot Discord", - "hostname": "5.199.136.34", - "username": "nonroot", - "port": "22", - "path": "/var/", - "remote_type": "sftp" - }, - "sftp_vm_discord_autism_bot_src_mysqldump":{ - "description": "sftp server for autismbot Discord", - "hostname": "5.199.136.34", - "username": "nonroot", - "port": "22", - "path": "", - "remote_type": "sftp" - }, - "sftp_vm_discord_autism_bot_src_autism_bot":{ - "description": "sftp server for autismbot Discord", - "hostname": "5.199.136.34", - "username": "nonroot", - "port": "22", - "path": "/home/nonroot/autism_bot/", - "remote_type": "sftp" - }, - "sftp_vm_backups_dest_ovh":{ - "description": "sftp server for backups, fastdl & demos", - "hostname": "5.199.136.34", - "username": "nonroot", - "port": "22", - "path": "/home/nonroot/backups/ovh/", - "remote_type": "sftp" - }, - "sftp_vm_backups2_dest_ovh":{ - "description": "Second sftp server for backups, fastdl & demos", - "hostname": "78.31.67.70", - "username": "autismbot5", - "port": "22", - "path": "/home/autismbot5/compressed_backups/ovh/", - "remote_type": "sftp" - }, - "sftp_vm_backups_dest_discord_autism_bot":{ - "description": "sftp server for discord autism bot", - "hostname": "5.199.136.34", - "username": "nonroot", - "port": "22", - "path": "/home/nonroot/backups/autism_bot/", - "remote_type": "sftp" - }, - "sftp_vm_backups2_dest_discord_autism_bot":{ - "description": "Second sftp server for discord autism bot", - "hostname": "78.31.67.70", - "username": "autismbot5", - "port": "22", - "path": "/home/autismbot5/compressed_backups/autism_bot/", - "remote_type": "sftp" - }, - "sftp_vm_backups_dest_hlstats":{ - "description": "sftp server for backups, fastdl & demos", - "hostname": "5.199.136.34", - "username": "nonroot", - "port": "22", - "path": "/home/nonroot/backups/hlstats/", - "remote_type": "sftp" - }, - "sftp_vm_backups2_dest_hlstats":{ - "description": "Second sftp server for backups, fastdl & demos", - "hostname": "78.31.67.70", - "username": "autismbot5", - "port": "22", - "path": "/home/autismbot5/compressed_backups/hlstats/", - "remote_type": "sftp" - }, - "sftp_vm_backups_dest_xenforo":{ - "description": "sftp server for backups, fastdl & demos", - "hostname": "5.199.136.34", - "username": "nonroot", - "port": "22", - "path": "/home/nonroot/backups/xenforo/", - "remote_type": "sftp" - }, - "sftp_vm_backups2_dest_xenforo":{ - "description": "Second sftp server for backups, fastdl & demos", - "hostname": "78.31.67.70", - "username": "autismbot5", - "port": "22", - "path": "/home/autismbot5/compressed_backups/xenforo/", - "remote_type": "sftp" - }, - "sftp_vm_backups_dest_webservices":{ - "description": "sftp server for backups, fastdl & demos", - "hostname": "5.199.136.34", - "username": "nonroot", - "port": "22", - "path": "/home/nonroot/backups/webservices/", - "remote_type": "sftp" - }, - "sftp_vm_backups2_dest_webservices":{ - "description": "Second sftp server for backups, fastdl & demos", - "hostname": "78.31.67.70", - "username": "autismbot5", - "port": "22", - "path": "/home/autismbot5/compressed_backups/webservices/", - "remote_type": "sftp" - } + "sftp_hetzner_src_home":{ + "description": "sftp server hetzner", + "hostname": "iphere", + "username": "nonroot", + "port": "22", + "path": "/home/", + "remote_type": "sftp" + }, + "sftp_hetzner_src_mysqldump":{ + "description": "sftp server hetzner", + "hostname": "iphere", + "username": "nonroot", + "port": "22", + "path": "", + "remote_type": "sftp" + }, + "sftp_dest_hetzner":{ + "description": "sftp server hetzner", + "hostname": "iphere", + "username": "nonroot", + "port": "22", + "path": "/home/nonroot/backups/", + "remote_type": "sftp" + }, + "sftp_dest_backups2":{ + "description": "Second sftp server", + "hostname": "iphere", + "username": "autismbot5", + "port": "22", + "path": "/home/autismbot5/backups/", + "remote_type": "sftp" + } }, "jobs":[ { - "job_name": "backup_remote_webservices_etc", - "job_description": "zips the remote webservices etc folder and moves it to backup/fastdl/demo vm", - "src": "sftp_vm_webservices_src_etc", - "dest": "sftp_vm_backups_dest_webservices", - "dest2": "sftp_vm_backups2_dest_webservices", - "zipname":"webservices_etc_backup", - "download_dir": "/home/file_mover/" - }, + "job_name": "backup_remote_hetzner_home", + "job_description": "zips hetzner home directory", + "src": "sftp_hetzner_src_home", + "dest": "sftp_dest_hetzner", + "dest2": "sftp_dest_backups2", + "zipname":"hetzner_home_dir", + "download_dir": "/home/file_mover/" + }, { - "job_name": "backup_remote_webservices_var", - "job_description": "zips the remote webservices var folder and moves it to backup/fastdl/demo vm", - "src": "sftp_vm_webservices_src_var", - "dest": "sftp_vm_backups_dest_webservices", - "dest2": "sftp_vm_backups2_dest_webservices", - "zipname":"webservices_var_backup", - "download_dir": "/home/file_mover/" - }, - { - "job_name": "backup_remote_webservices_home", - "job_description": "zips the remote webservices home folder and moves it to backup/fastdl/demo vm", - "src": "sftp_vm_webservices_src_home", - "dest": "sftp_vm_backups_dest_webservices", - "dest2": "sftp_vm_backups2_dest_webservices", - "zipname":"webservices_home_backup", - "download_dir": "/home/file_mover/" - }, - { - "job_name": "backup_remote_webservices_mysqldump", - "job_description": "dumps the remote webservices vm databases and moves them to backup/fastdl/demo vm", - "src": "sftp_vm_webservices_src_mysqldump", - "dest": "sftp_vm_backups_dest_webservices", - "dest2": "sftp_vm_backups2_dest_webservices", - "zipname":"webservices_mysqldump_backup", - "download_dir": "/home/file_mover/" - }, - { - "job_name": "backup_remote_xenforo_etc", - "job_description": "zips the remote xenforo etc folder and moves it to backup/fastdl/demo vm", - "src": "sftp_vm_xenforo_src_etc", - "dest": "sftp_vm_backups_dest_xenforo", - "dest2": "sftp_vm_backups2_dest_xenforo", - "zipname":"xenforo_etc_backup", - "download_dir": "/home/file_mover/" - }, - { - "job_name": "backup_remote_xenforo_var", - "job_description": "zips the remote xenforo var folder and moves it to backup/fastdl/demo vm", - "src": "sftp_vm_xenforo_src_var", - "dest": "sftp_vm_backups_dest_xenforo", - "dest2": "sftp_vm_backups2_dest_xenforo", - "zipname":"xenforo_var_backup", - "download_dir": "/home/file_mover/" - }, - { - "job_name": "backup_remote_xenforo_home", - "job_description": "zips the remote xenforo home folder and moves it to backup/fastdl/demo vm", - "src": "sftp_vm_xenforo_src_home", - "dest": "sftp_vm_backups_dest_xenforo", - "dest2": "sftp_vm_backups2_dest_xenforo", - "zipname":"xenforo_home_backup", - "download_dir": "/home/file_mover/" - }, - { - "job_name": "backup_remote_xenforo_mysqldump", - "job_description": "dumps the remote xenforo vm databases and moves them to backup/fastdl/demo vm", - "src": "sftp_vm_xenforo_src_mysqldump", - "dest": "sftp_vm_backups_dest_xenforo", - "dest2": "sftp_vm_backups2_dest_xenforo", - "zipname":"xenforo_mysqldump_backup", - "download_dir": "/home/file_mover/" - }, - { - "job_name": "backup_remote_hlstats_etc", - "job_description": "zips the remote hlstats etc folder and moves it to backup/fastdl/demo vm", - "src": "sftp_vm_hlstats_src_etc", - "dest": "sftp_vm_backups_dest_hlstats", - "dest2": "sftp_vm_backups2_dest_hlstats", - "zipname":"hlstats_etc_backup", - "download_dir": "/home/file_mover/" - }, - { - "job_name": "backup_remote_hlstats_var", - "job_description": "zips the remote hlstats var folder and moves it to backup/fastdl/demo vm", - "src": "sftp_vm_hlstats_src_var", - "dest": "sftp_vm_backups_dest_hlstats", - "dest2": "sftp_vm_backups2_dest_hlstats", - "zipname":"hlstats_var_backup", - "download_dir": "/home/file_mover/" - }, - { - "job_name": "backup_remote_hlstats_mysqldump", - "job_description": "dumps the remote hlstats vm databases and moves them to backup/fastdl/demo vm", - "src": "sftp_vm_hlstats_src_mysqldump", - "dest": "sftp_vm_backups_dest_hlstats", - "dest2": "sftp_vm_backups2_dest_hlstats", - "zipname":"hlstats_mysqldump_backup", - "download_dir": "/home/file_mover/" - }, - { - "job_name": "backup_remote_hlstats_home", - "job_description": "zips the remote hlstats home folder and moves it to backup/fastdl/demo vm", - "src": "sftp_vm_hlstats_src_home", - "dest": "sftp_vm_backups_dest_hlstats", - "dest2": "sftp_vm_backups2_dest_hlstats", - "zipname":"hlstats_home_backup", - "download_dir": "/home/file_mover/" - }, - { - "job_name": "backup_remote_autism_bot_etc", - "job_description": "zips the remote discord autism bot etc folder and moves it to discord autism bot vm", - "src": "sftp_vm_discord_autism_bot_src_etc", - "dest": "sftp_vm_backups_dest_discord_autism_bot", - "dest2": "sftp_vm_backups2_dest_discord_autism_bot", - "zipname":"discord_autism_bot_etc_backup", - "download_dir": "/home/file_mover/" - }, - { - "job_name": "backup_remote_autism_bot_var", - "job_description": "zips the remote discord autism bot var folder and moves it to discord autism bot vm", - "src": "sftp_vm_discord_autism_bot_src_var", - "dest": "sftp_vm_backups_dest_discord_autism_bot", - "dest2": "sftp_vm_backups2_dest_discord_autism_bot", - "zipname":"discord_autism_bot_var_backup", - "download_dir": "/home/file_mover/" - }, - { - "job_name": "backup_remote_autism_bot_mysqldump", - "job_description": "dumps the remote discord autism bot vm databases and moves them to discord autism bot vm", - "src": "sftp_vm_discord_autism_bot_src_mysqldump", - "dest": "sftp_vm_backups_dest_discord_autism_bot", - "dest2": "sftp_vm_backups2_dest_discord_autism_bot", - "zipname":"discord_autism_bot_mysqldump_backup", - "download_dir": "/home/file_mover/" - }, - { - "job_name": "backup_remote_autism_bot_discord", - "job_description": "zips the remote autism bot discord autism bot discord folder and moves it to backup/demo vm", - "src": "sftp_vm_discord_autism_bot_src_autism_bot", - "dest": "sftp_vm_backups_dest_discord_autism_bot", - "dest2": "sftp_vm_backups2_dest_discord_autism_bot", - "zipname":"discord_autism_bot_folder_backup", - "download_dir": "/home/file_mover/" - }, + "job_name": "hetzner_mysqldump", + "job_description": "dumps hetzner databases", + "src": "sftp_hetzner_src_mysqldump", + "dest": "sftp_dest_hetzner", + "dest2": "sftp_dest_backups2", + "zipname":"hetzner_mysqldump", + "download_dir": "/home/file_mover/" + }, { "job_name": "backup_local_gameservers", - "job_description": "zips the local gameservers folder and moves it to backup/fastdl/demo vm", + "job_description": "zips the local gameservers folder", "src": "local_dir_gameservers", - "dest": "sftp_vm_backups_dest_ovh", - "dest2": "sftp_vm_backups2_dest_ovh", - "zipname":"ovh_gameservers_backup" - }, - { - "job_name": "backup_local_var", - "job_description": "zips the local var folder and moves it to backup/fastdl/demo vm", - "src": "local_dir_var", - "dest": "sftp_vm_backups_dest_ovh", - "dest2": "sftp_vm_backups2_dest_ovh", - "zipname":"ovh_var_backup" - }, - { - "job_name": "backup_local_etc", - "job_description": "zips the local etc folder and moves it to backup/fastdl/demo vm", - "src": "local_dir_etc", - "dest": "sftp_vm_backups_dest_ovh", - "dest2": "sftp_vm_backups2_dest_ovh", - "zipname":"ovh_etc_backup" - }, - { - "job_name": "backup_local_mysqldump", - "job_description": "zips the local databases and moves it to backup/fastdl/demo vm", - "src": "local_dir_mysqldump", - "dest": "sftp_vm_backups_dest_ovh", - "dest2": "sftp_vm_backups2_dest_ovh", - "zipname":"ovh_mysqldump_backup" - } + "dest": "sftp_dest_hetzner", + "dest2": "sftp_dest_backups2", + "zipname":"ovh_gameservers" + } ], "settings": { "sftp": { "remote_attempts": "5", "remote_delay": "15", - "5.199.136.34":{ - "username": { - "password": "" - } - }, - "5.199.136.22":{ - "username": { - "password": "" - } - }, - "5.104.107.130":{ - "username": { - "password": "" - } - }, - "5.104.107.39":{ - "username": { - "password": "" - } - }, - "78.31.67.70": { - "username": { + "iphere":{ + "nonroot": { + "password": "" + } + }, + "iphere": { + "autismbot5": { "password": "" } } diff --git a/file_mover/file_mover.py b/file_mover/file_mover.py index 142d723e..b1e92060 100755 --- a/file_mover/file_mover.py +++ b/file_mover/file_mover.py @@ -45,8 +45,8 @@ def load_config(config_file): sys.exit(1) def main(): - #runs demos every 5 minutes but backups thrice a week with systemctl services/timers - config_file = 'config_demos.json' + #runs backups + config_file = 'config_backups.json' if sys.argv[1:]: config_file = sys.argv[1] remotes, jobs, settings = load_config(config_file) @@ -56,15 +56,9 @@ def main(): dest2 = create_remote(remotes[job["dest2"]], settings) if index == 0: #delete old files (not very adaptable, cant handle dirs in dirs) - dest.delete_remote_zips("/home/nonroot/backups") - dest2.delete_remote_zips("/home/autismbot5/compressed_backups") - if 'demo_' in job["job_name"]: - source_files = src.list_dir() - distribute_files(source_files, dest) - for pathfile in source_files: - src.delete_local_demo(pathfile) - dest.delete_remote_demos(source_files) - elif "backup_local" in job["job_name"]: + dest.delete_remote_zips("/home/nonroot/backups/") + dest2.delete_remote_zips("/home/autismbot5/backups/") + if "backup_local" in job["job_name"]: zip_file_path = src.zip_local_directory(job) if not dest.put(zip_file_path, dest.path): log_msg = ''.join(["failed putting file: ", str(pathfile)]) @@ -120,3 +114,4 @@ def main(): if __name__ == '__main__': main() + diff --git a/file_mover/remote_local_dir.py b/file_mover/remote_local_dir.py index ade303e0..321b23eb 100755 --- a/file_mover/remote_local_dir.py +++ b/file_mover/remote_local_dir.py @@ -1,7 +1,3 @@ -from pathlib import Path -import contextlib -import os -from datetime import timedelta, datetime import subprocess class local_dir_remote: diff --git a/file_mover/remote_sftp.py b/file_mover/remote_sftp.py index 5e1af7bc..7ee4e751 100755 --- a/file_mover/remote_sftp.py +++ b/file_mover/remote_sftp.py @@ -50,11 +50,12 @@ class sftp_remote: channel = self.ssh.get_transport().open_session(timeout=120) #-T3= 3 threads, use 70% of available ram + #TODO exclude dir where hlstats spams pictures of the player graphs if 'tar_recursive' == command_state: - exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar --exclude={zipname} -cJf {zipname} {self.path}" + exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar --exclude={zipname} --exclude=/home/nonroot/backups/ --exclude=/home/nonroot/demos --exclude=/home/nonroot/snap --exclude=logs/ --ignore-failed-read -cJf {zipname} {self.path}" elif 'mysqldump' == command_state: #check README for my.cnf regarding how this works - exec_command_state = f'mysqldump -u root --all-databases > mysqldump.sql' + exec_command_state = f'mysqldump -u backups_db --all-databases > mysqldump.sql' else: #tar_mysqldump exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar -cJf {zipname} /home/{self.username}/mysqldump.sql" exec_cmd = f'cd /home/{self.username}/; {exec_command_state}' @@ -187,14 +188,13 @@ class sftp_remote: def delete_remote_zips(self, backups_dir): self.connect() - for directories in self.sftp.listdir(backups_dir): - for files in self.sftp.listdir(f'{backups_dir}/{directories}'): - pathfile = f'{backups_dir}/{directories}/{files}' - if not pathfile.endswith(".xz"): - continue - utime = self.sftp.stat(pathfile).st_mtime - last_modified = datetime.fromtimestamp(utime) - self.delete_file(last_modified, pathfile, 25) + for files in self.sftp.listdir(f'{backups_dir}'): + pathfile = f'{backups_dir}{files}' + if not pathfile.endswith(".xz"): + continue + utime = self.sftp.stat(pathfile).st_mtime + last_modified = datetime.fromtimestamp(utime) + self.delete_file(last_modified, pathfile, 40) self.disconnect() def delete_remote_zip_temp(self, zipname):