diff --git a/file_mover/config_backups.json b/file_mover/config_backups.json index 3a1450d8..b6baf90c 100755 --- a/file_mover/config_backups.json +++ b/file_mover/config_backups.json @@ -15,6 +15,118 @@ "path": "/etc/", "remote_type": "local_dir" }, + "sftp_vm_hlstats_src_home":{ + "description": "sftp server for hlstats", + "hostname": "51.15.159.31", + "username": "nonroot", + "port": "22", + "path": "/home/", + "remote_type": "sftp" + }, + "sftp_vm_hlstats_src_etc":{ + "description": "sftp server for hlstats", + "hostname": "51.15.159.31", + "username": "nonroot", + "port": "22", + "path": "/etc/", + "remote_type": "sftp" + }, + "sftp_vm_hlstats_src_var":{ + "description": "sftp server for hlstats", + "hostname": "51.15.159.31", + "username": "nonroot", + "port": "22", + "path": "/var/", + "remote_type": "sftp" + }, + "sftp_vm_hlstats_src_mysqldump":{ + "description": "sftp server for hlstats", + "hostname": "51.15.159.31", + "username": "nonroot", + "port": "22", + "path": "", + "remote_type": "sftp" + }, + "sftp_vm_xenforo_src_home":{ + "description": "sftp server for xenforo", + "hostname": "163.172.225.175", + "username": "nonroot", + "port": "22", + "path": "/home/", + "remote_type": "sftp" + }, + "sftp_vm_xenforo_src_etc":{ + "description": "sftp server for xenforo", + "hostname": "163.172.225.175", + "username": "nonroot", + "port": "22", + "path": "/etc/", + "remote_type": "sftp" + }, + "sftp_vm_xenforo_src_var":{ + "description": "sftp server for xenforo", + "hostname": "163.172.225.175", + "username": "nonroot", + "port": "22", + "path": "/var/", + "remote_type": "sftp" + }, + "sftp_vm_xenforo_src_mysqldump":{ + "description": "sftp server for xenforo", + "hostname": "163.172.225.175", + "username": "nonroot", + "port": "22", + "path": "", + "remote_type": "sftp" + }, + "sftp_vm_webservices_src_home":{ + "description": "sftp server for webservices", + "hostname": "163.172.117.46", + "username": "nonroot", + "port": "22", + "path": "/home/", + "remote_type": "sftp" + }, + "sftp_vm_webservices_src_etc":{ + "description": "sftp server for webservices", + "hostname": "163.172.117.46", + "username": "nonroot", + "port": "22", + "path": "/etc/", + "remote_type": "sftp" + }, + "sftp_vm_webservices_src_var":{ + "description": "sftp server for webservices", + "hostname": "163.172.117.46", + "username": "nonroot", + "port": "22", + "path": "/var/", + "remote_type": "sftp" + }, + "sftp_vm_webservices_src_mysqldump":{ + "description": "sftp server for webservices", + "hostname": "163.172.117.46", + "username": "nonroot", + "port": "22", + "path": "", + "remote_type": "sftp" + }, + "sftp_vm_backups_src_etc":{ + "description": "sftp server for backups/fastdl/demos", + "hostname": "163.172.119.53", + "username": "nonroot", + "port": "22", + "path": "/etc/", + "remote_type": "sftp" + }, + "sftp_vm_backups_src_var":{ + "description": "sftp server for backups/fastdl/demos", + "hostname": "163.172.119.53", + "username": "nonroot", + "port": "22", + "path": "/var/", + "remote_type": "sftp" + }, "sftp_vm_backups_dest_ovh":{ "description": "sftp server for backups, fastdl & demos", "hostname": "163.172.119.53", @@ -22,9 +134,150 @@ "port": "22", "path": "/home/nonroot/backups/ovh/", "remote_type": "sftp" + }, + "sftp_vm_backups_dest_fastdl":{ + "description": "sftp server for backups, fastdl & demos", + "hostname": "163.172.119.53", + "username": "nonroot", + "port": "22", + "path": "/home/nonroot/backups/fastdl/", + "remote_type": "sftp" + }, + "sftp_vm_backups_dest_hlstats":{ + "description": "sftp server for backups, fastdl & demos", + "hostname": "163.172.119.53", + "username": "nonroot", + "port": "22", + "path": "/home/nonroot/backups/hlstats/", + "remote_type": "sftp" + }, + "sftp_vm_backups_dest_xenforo":{ + "description": "sftp server for backups, fastdl & demos", + "hostname": "163.172.119.53", + "username": "nonroot", + "port": "22", + "path": "/home/nonroot/backups/xenforo/", + "remote_type": "sftp" + }, + "sftp_vm_backups_dest_webservices":{ + "description": "sftp server for backups, fastdl & demos", + "hostname": "163.172.119.53", + "username": "nonroot", + "port": "22", + "path": "/home/nonroot/backups/webservices/", + "remote_type": "sftp" } }, "jobs":[ + { + "job_name": "backup_remote_webservices_etc", + "job_description": "zips the remote webservices etc folder and moves it to backup/fastdl/demo vm", + "src": "sftp_vm_webservices_src_etc", + "dest": "sftp_vm_backups_dest_webservices", + "zipname":"webservices_etc_backup", + "download_dir": "/home/file_mover/" + }, + { + "job_name": "backup_remote_webservices_var", + "job_description": "zips the remote webservices var folder and moves it to backup/fastdl/demo vm", + "src": "sftp_vm_webservices_src_var", + "dest": "sftp_vm_backups_dest_webservices", + "zipname":"webservices_var_backup", + "download_dir": "/home/file_mover/" + }, + { + "job_name": "backup_remote_webservices_home", + "job_description": "zips the remote webservices home folder and moves it to backup/fastdl/demo vm", + "src": "sftp_vm_webservices_src_home", + "dest": "sftp_vm_backups_dest_webservices", + "zipname":"webservices_home_backup", + "download_dir": "/home/file_mover/" + }, + { + "job_name": "backup_remote_webservices_mysqldump", + "job_description": "dumps the remote webservices vm databases and moves them to backup/fastdl/demo vm", + "src": "sftp_vm_webservices_src_mysqldump", + "dest": "sftp_vm_backups_dest_webservices", + "zipname":"webservices_mysqldump_backup" + }, + { + "job_name": "backup_remote_xenforo_etc", + "job_description": "zips the remote xenforo etc folder and moves it to backup/fastdl/demo vm", + "src": "sftp_vm_xenforo_src_etc", + "dest": "sftp_vm_backups_dest_xenforo", + "zipname":"xenforo_etc_backup", + "download_dir": "/home/file_mover/" + }, + { + "job_name": "backup_remote_xenforo_var", + "job_description": "zips the remote xenforo var folder and moves it to backup/fastdl/demo vm", + "src": "sftp_vm_xenforo_src_var", + "dest": "sftp_vm_backups_dest_xenforo", + "zipname":"xenforo_var_backup", + "download_dir": "/home/file_mover/" + }, + { + "job_name": "backup_remote_xenforo_home", + "job_description": "zips the remote xenforo home folder and moves it to backup/fastdl/demo vm", + "src": "sftp_vm_xenforo_src_home", + "dest": "sftp_vm_backups_dest_xenforo", + "zipname":"xenforo_home_backup", + "download_dir": "/home/file_mover/" + }, + { + "job_name": "backup_remote_xenforo_mysqldump", + "job_description": "dumps the remote xenforo vm databases and moves them to backup/fastdl/demo vm", + "src": "sftp_vm_xenforo_src_mysqldump", + "dest": "sftp_vm_backups_dest_xenforo", + "zipname":"xenforo_mysqldump_backup" + }, + { + "job_name": "backup_remote_hlstats_etc", + "job_description": "zips the remote hlstats etc folder and moves it to backup/fastdl/demo vm", + "src": "sftp_vm_hlstats_src_etc", + "dest": "sftp_vm_backups_dest_hlstats", + "zipname":"hlstats_etc_backup", + "download_dir": "/home/file_mover/" + }, + { + "job_name": "backup_remote_hlstats_var", + "job_description": "zips the remote hlstats var folder and moves it to backup/fastdl/demo vm", + "src": "sftp_vm_hlstats_src_var", + "dest": "sftp_vm_backups_dest_hlstats", + "zipname":"hlstats_var_backup", + "download_dir": "/home/file_mover/" + }, + { + "job_name": "backup_remote_hlstats_mysqldump", + "job_description": "dumps the remote hlstats vm databases and moves them to backup/fastdl/demo vm", + "src": "sftp_vm_hlstats_src_mysqldump", + "dest": "sftp_vm_backups_dest_hlstats", + "zipname":"hlstats_mysqldump_backup" + }, + { + "job_name": "backup_remote_hlstats_home", + "job_description": "zips the remote hlstats home folder and moves it to backup/fastdl/demo vm", + "src": "sftp_vm_hlstats_src_home", + "dest": "sftp_vm_backups_dest_hlstats", + "zipname":"hlstats_home_backup", + "download_dir": "/home/file_mover/" + }, + { + "job_name": "backup_remote_fastdl_etc", + "job_description": "zips the remote backups/fastdl/demos etc folder and moves it to backup/fastdl/demo vm", + "src": "sftp_vm_backups_src_etc", + "dest": "sftp_vm_backups_dest_fastdl", + "zipname":"fastdl_etc_backup", + "download_dir": "/home/file_mover/" + }, + { + "job_name": "backup_remote_fastdl_var", + "job_description": "zips the remote backups/fastdl/demos var folder and moves it to backup/fastdl/demo vm", + "src": "sftp_vm_backups_src_var", + "dest": "sftp_vm_backups_dest_fastdl", + "zipname":"fastdl_var_backup", + "download_dir": "/home/file_mover/" + }, { "job_name": "backup_local_gameservers", "job_description": "zips the local gameservers folder and moves it to backup/fastdl/demo vm", @@ -51,11 +304,26 @@ "sftp": { "remote_attempts": "5", "remote_delay": "15", - "ip address":{ + "ip":{ "username": { "password": "password" } - } + }, + "ip":{ + "username": { + "password": "password" + } + }, + "ip":{ + "username": { + "password": "password" + } + }, + "ip":{ + "password": { + "password": "password" + } + } } } } diff --git a/file_mover/file_mover.py b/file_mover/file_mover.py index c657e92f..9f9557c2 100755 --- a/file_mover/file_mover.py +++ b/file_mover/file_mover.py @@ -67,7 +67,24 @@ def main(): sys.exit(1) src.delete_local_zip(zip_file_path) dest.delete_remote_zips(zip_file_path) - + elif "mysqldump" in job["job_name"]: + pass + elif "backup_remote" in job["job_name"]: + zipname = src.zip_remote_directory(job) + if zipname is None: + logging.warning(f'failed zipping remote directory at job {job}') + sys.exit(1) + local_zip_path_name = src.get_remote_files(job, zipname) + if local_zip_path_name is None: + logging.warning(f'failed getting remote zip at job {job}') + src.delete_remote_zip_temp(zipname) + sys.exit(1) + src.delete_remote_zip_temp(zipname) + if not dest.put(local_zip_path_name, dest.path): + logging.warning(f'failed putting local zip {local_zip_path_name} at job {job}') + src.delete_local_zip(local_zip_path_name) + sys.exit(1) + src.delete_local_zip(local_zip_path_name) logging.info(('finished job: ', job)) if __name__ == '__main__': diff --git a/file_mover/remote_sftp.py b/file_mover/remote_sftp.py index 090aa132..5071c968 100755 --- a/file_mover/remote_sftp.py +++ b/file_mover/remote_sftp.py @@ -1,7 +1,6 @@ import paramiko from paramiko.ssh_exception import SSHException, NoValidConnectionsError import time -import sys import hashlib import stat from datetime import datetime, timedelta @@ -20,6 +19,7 @@ class sftp_remote: self.remote_delay = settings[self.remote_type]['remote_delay'] self.remote_error = None self.password = settings[self.remote_type][self.hostname][self.username]['password'] + self.ssh = None def connect(self): self.transport = paramiko.Transport((self.hostname, int(self.port))) @@ -34,6 +34,64 @@ class sftp_remote: del (self.sftp) self.transport.close() + def ssh_connect(self): + self.ssh = paramiko.SSHClient() + #i trust my vm's + self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.ssh.connect(self.hostname, username=self.username, password=self.password) + + def ssh_disconnect(self): + self.ssh.close() + + def zip_remote_directory(self, job): + zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.zip' + self.ssh_connect() + channel = self.ssh.get_transport().open_session(timeout=120) + exec_cmd = f'cd /home/{self.username}/; zip -r {zipname} {self.path}' + keyboard_interrupt = False + try: + channel.exec_command(exec_cmd) + #2^21 (2,097,152) characters before filling up the buffers of exec_command + while True: + buf = channel.recv(1024) + if not buf: + break + #priting the output for sure causes it to run a bit slower + print('buffer: ', buf) + channel.recv_exit_status() + except KeyboardInterrupt: + keyboard_interrupt = True + finally: + channel.close() + self.ssh_disconnect() + if keyboard_interrupt: + print("manually interrupted") + sys.exit(1) + return zipname + + def get_remote_files(self, job, zipname): + total_attempts = int(self.remote_attempts) + while total_attempts > 0: + self.connect() + job_path = job["download_dir"] + self.sftp.get(f'/home/{self.username}/{zipname}', f'{job_path}{zipname}') + sha256_first = self.digest(f'{job_path}{zipname}') + os.remove(f'{job_path}{zipname}') + self.disconnect() + + #redownloading to validate the SHA sum + self.connect() + self.sftp.get(f'/home/{self.username}/{zipname}', f'{job_path}{zipname}') + sha256_second = self.digest(f'{job_path}{zipname}') + self.disconnect() + if sha256_first == sha256_second: + return f'{job_path}{zipname}' + os.remove(f'{job_path}{zipname}') + total_attempts = self.subtract_remote_attempts(total_attempts) + if total_attempts == 0: + return None + + def subtract_remote_attempts(self, total_attempts): time.sleep(self.remote_delay) return total_attempts -1 @@ -115,3 +173,13 @@ class sftp_remote: pathfile, last_modified = self.change_path(zip_file_path) self.delete_file(last_modified, pathfile, 10) self.disconnect() + + def delete_remote_zip_temp(self, zipname): + self.connect() + self.sftp.remove(f'/home/{self.username}/{zipname}') + self.disconnect() + + def delete_local_zip(self, local_zip_path_name): + os.remove(local_zip_path_name) + +