From 9ae3ddfcd6b813aabed2491ad502b522eec76f85 Mon Sep 17 00:00:00 2001 From: christian Date: Tue, 2 Nov 2021 21:36:38 +0100 Subject: [PATCH] added mysqldump support --- file_mover/README.md | 6 ++++++ file_mover/config_backups.json | 11 +++++++---- file_mover/file_mover.py | 19 +++++++++++++++++-- file_mover/remote_local_dir.py | 1 + file_mover/remote_sftp.py | 17 +++++++++++++++-- file_mover/systemctl/demo_mover.service | 1 + 6 files changed, 47 insertions(+), 8 deletions(-) diff --git a/file_mover/README.md b/file_mover/README.md index 306ce64f..d99c1355 100755 --- a/file_mover/README.md +++ b/file_mover/README.md @@ -10,6 +10,12 @@ ##globally: apt-get install zip +##requires an ~/.my.cnf with + +[mysqldump] +host="" +user="" +password="" ##python3 file_mover.py config.json diff --git a/file_mover/config_backups.json b/file_mover/config_backups.json index b6baf90c..d9d42bd4 100755 --- a/file_mover/config_backups.json +++ b/file_mover/config_backups.json @@ -198,7 +198,8 @@ "job_description": "dumps the remote webservices vm databases and moves them to backup/fastdl/demo vm", "src": "sftp_vm_webservices_src_mysqldump", "dest": "sftp_vm_backups_dest_webservices", - "zipname":"webservices_mysqldump_backup" + "zipname":"webservices_mysqldump_backup", + "download_dir": "/home/file_mover/" }, { "job_name": "backup_remote_xenforo_etc", @@ -229,7 +230,8 @@ "job_description": "dumps the remote xenforo vm databases and moves them to backup/fastdl/demo vm", "src": "sftp_vm_xenforo_src_mysqldump", "dest": "sftp_vm_backups_dest_xenforo", - "zipname":"xenforo_mysqldump_backup" + "zipname":"xenforo_mysqldump_backup", + "download_dir": "/home/file_mover/" }, { "job_name": "backup_remote_hlstats_etc", @@ -252,7 +254,8 @@ "job_description": "dumps the remote hlstats vm databases and moves them to backup/fastdl/demo vm", "src": "sftp_vm_hlstats_src_mysqldump", "dest": "sftp_vm_backups_dest_hlstats", - "zipname":"hlstats_mysqldump_backup" + "zipname":"hlstats_mysqldump_backup", + "download_dir": "/home/file_mover/" }, { "job_name": "backup_remote_hlstats_home", @@ -320,7 +323,7 @@ } }, "ip":{ - "password": { + "username": { "password": "password" } } diff --git a/file_mover/file_mover.py b/file_mover/file_mover.py index 9f9557c2..d0fc7d51 100755 --- a/file_mover/file_mover.py +++ b/file_mover/file_mover.py @@ -68,9 +68,24 @@ def main(): src.delete_local_zip(zip_file_path) dest.delete_remote_zips(zip_file_path) elif "mysqldump" in job["job_name"]: - pass + src.remote_channel_command(job, "mysqldump") + zipname = src.remote_channel_command(job, "zip_mysqldump") + if zipname is None: + logging.warning(f'failed zipping remote directory at job {job}') + sys.exit(1) + local_zip_path_name = src.get_remote_files(job, zipname) + if local_zip_path_name is None: + logging.warning(f'failed getting remote zip at job {job}') + src.delete_remote_zip_temp(zipname) + sys.exit(1) + src.delete_remote_zip_temp(zipname) + if not dest.put(local_zip_path_name, dest.path): + logging.warning(f'failed putting local zip {local_zip_path_name} at job {job}') + src.delete_local_zip(local_zip_path_name) + sys.exit(1) + src.delete_local_zip(local_zip_path_name) elif "backup_remote" in job["job_name"]: - zipname = src.zip_remote_directory(job) + zipname = src.remote_channel_command(job, "zip_recursive") if zipname is None: logging.warning(f'failed zipping remote directory at job {job}') sys.exit(1) diff --git a/file_mover/remote_local_dir.py b/file_mover/remote_local_dir.py index f25cb4b9..9ff41162 100755 --- a/file_mover/remote_local_dir.py +++ b/file_mover/remote_local_dir.py @@ -16,6 +16,7 @@ class local_dir_remote: glob_pattern = '*' glob_pattern_recursive = '**' if not recursive_search: + #os. for file_demo in list(path.glob(glob_pattern)): mtime = datetime.fromtimestamp(os.stat(file_demo).st_mtime) if mtime < datetime.now() - delta: diff --git a/file_mover/remote_sftp.py b/file_mover/remote_sftp.py index 5071c968..b35e1b7d 100755 --- a/file_mover/remote_sftp.py +++ b/file_mover/remote_sftp.py @@ -3,6 +3,7 @@ from paramiko.ssh_exception import SSHException, NoValidConnectionsError import time import hashlib import stat +import sys from datetime import datetime, timedelta import os @@ -43,11 +44,18 @@ class sftp_remote: def ssh_disconnect(self): self.ssh.close() - def zip_remote_directory(self, job): + def remote_channel_command(self, job, command_state): zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.zip' self.ssh_connect() channel = self.ssh.get_transport().open_session(timeout=120) - exec_cmd = f'cd /home/{self.username}/; zip -r {zipname} {self.path}' + if 'zip_recursive' == command_state: + exec_command_state = f'zip -r {zipname} {self.path}' + elif 'mysqldump' == command_state: + #check README for my.cnf regarding how this works + exec_command_state = f'mysqldump -u root --all-databases > mysqldump.sql' + else: #zip_mysqldump + exec_command_state = f'zip {zipname} /home/{self.username}/mysqldump.sql' + exec_cmd = f'cd /home/{self.username}/; {exec_command_state}' keyboard_interrupt = False try: channel.exec_command(exec_cmd) @@ -63,6 +71,11 @@ class sftp_remote: keyboard_interrupt = True finally: channel.close() + if 'zip_mysqldump' == command_state: + file_path = f'/home/{self.username}/mysqldump.sql' + self.connect() + self.sftp.remove(file_path) + self.disconnect() self.ssh_disconnect() if keyboard_interrupt: print("manually interrupted") diff --git a/file_mover/systemctl/demo_mover.service b/file_mover/systemctl/demo_mover.service index b453628b..b5075f5e 100755 --- a/file_mover/systemctl/demo_mover.service +++ b/file_mover/systemctl/demo_mover.service @@ -3,6 +3,7 @@ Description=Moves demos from the local machine to remote destinations every 5 mi [Service] Type=simple +#the ovh gameservers uses the gameserver user and /home/gameservers/demos instead due to permission problems with auto generated demos User=file_mover WorkingDirectory=/home/file_mover ExecStart=/home/file_mover/run_demo_mover.sh