diff --git a/file_mover/file_mover.py b/file_mover/file_mover.py index 816bea74..bc09b350 100755 --- a/file_mover/file_mover.py +++ b/file_mover/file_mover.py @@ -71,7 +71,7 @@ def main(): src.delete_local_zip(zip_file_path) elif "mysqldump" in job["job_name"]: src.remote_channel_command(job, "mysqldump") - zipname = src.remote_channel_command(job, "zip_mysqldump") + zipname = src.remote_channel_command(job, "tar_mysqldump") if zipname is None: logging.warning(f'failed zipping remote directory at job {job}') sys.exit(1) @@ -87,7 +87,7 @@ def main(): sys.exit(1) src.delete_local_zip(local_zip_path_name) elif "backup_remote" in job["job_name"]: - zipname = src.remote_channel_command(job, "zip_recursive") + zipname = src.remote_channel_command(job, "tar_recursive") if zipname is None: logging.warning(f'failed zipping remote directory at job {job}') sys.exit(1) diff --git a/file_mover/remote_local_dir.py b/file_mover/remote_local_dir.py index fbeb34bd..cbe9e6de 100755 --- a/file_mover/remote_local_dir.py +++ b/file_mover/remote_local_dir.py @@ -2,7 +2,7 @@ from pathlib import Path import contextlib import os from datetime import timedelta, datetime -from zipfile import ZipFile +import subprocess class local_dir_remote: def __init__(self, config): @@ -34,13 +34,12 @@ class local_dir_remote: def zip_local_directory(self, job): source_files = self.list_dir(True) - zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.zip' - with ZipFile(zipname, 'w') as myzip: - for files in source_files: - try: - myzip.write(files) - except Exception: - continue + zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.tar.xz' + #-T3 = 3 threads, the backups are meant to run in the night at 5 am to not interfere with players. + exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar -cJf {zipname} {self.path}" + print("exec_command_state: ", exec_command_state) + subprocess.run([exec_command_state], shell=True) + return str(Path.cwd()) + '/' + zipname def delete_local_demo(self, pathfile): @@ -51,6 +50,6 @@ class local_dir_remote: def delete_local_zip(self, zip_file_path): with contextlib.suppress(FileNotFoundError): - if (str(zip_file_path).endswith('.zip')): + if (str(zip_file_path).endswith('.tar.xz')): os.remove(zip_file_path) - print('removed zip: ', zip_file_path) + print('removed tar xz: ', zip_file_path) diff --git a/file_mover/remote_sftp.py b/file_mover/remote_sftp.py index 08c71af2..39482b97 100755 --- a/file_mover/remote_sftp.py +++ b/file_mover/remote_sftp.py @@ -45,16 +45,18 @@ class sftp_remote: self.ssh.close() def remote_channel_command(self, job, command_state): - zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.zip' + zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.tar.xz' self.ssh_connect() channel = self.ssh.get_transport().open_session(timeout=120) - if 'zip_recursive' == command_state: - exec_command_state = f'zip -r {zipname} {self.path}' + + #-T3= 3 threads, use 70% of available ram + if 'tar_recursive' == command_state: + exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar -cJf {zipname} {self.path}" elif 'mysqldump' == command_state: #check README for my.cnf regarding how this works exec_command_state = f'mysqldump -u root --all-databases > mysqldump.sql' - else: #zip_mysqldump - exec_command_state = f'zip {zipname} /home/{self.username}/mysqldump.sql' + else: #tar_mysqldump + exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar -cJf {zipname} /home/{self.username}/mysqldump.sql" exec_cmd = f'cd /home/{self.username}/; {exec_command_state}' keyboard_interrupt = False try: @@ -65,13 +67,13 @@ class sftp_remote: if not buf: break #priting the output for sure causes it to run a bit slower - print('buffer: ', buf) + #print('buffer: ', buf) channel.recv_exit_status() except KeyboardInterrupt: keyboard_interrupt = True finally: channel.close() - if 'zip_mysqldump' == command_state: + if 'tar_mysqldump' == command_state: file_path = f'/home/{self.username}/mysqldump.sql' self.connect() self.sftp.remove(file_path) @@ -85,8 +87,11 @@ class sftp_remote: def get_remote_files(self, job, zipname): total_attempts = int(self.remote_attempts) while total_attempts > 0: + print(f'job: {job}') self.connect() job_path = job["download_dir"] + #print(f'/home/{self.username}/{zipname}') + #print(f'{job_path}{zipname}') self.sftp.get(f'/home/{self.username}/{zipname}', f'{job_path}{zipname}') sha256_first = self.digest(f'{job_path}{zipname}') os.remove(f'{job_path}{zipname}') @@ -199,4 +204,3 @@ class sftp_remote: def delete_local_zip(self, local_zip_path_name): os.remove(local_zip_path_name) - diff --git a/file_mover/systemctl/backups.timer b/file_mover/systemctl/backups.timer index 7f6e77e6..ac6270f0 100755 --- a/file_mover/systemctl/backups.timer +++ b/file_mover/systemctl/backups.timer @@ -3,7 +3,7 @@ Description=Creates backups thrice a week Requires=backups.service [Timer] -OnCalendar=Mon,Thu,Sat 10:00 +OnCalendar=Mon,Thu,Sat 04:00 [Install] WantedBy=multi-user.target