updated to xz compression

file_mover/file_mover.py

@@ -71,7 +71,7 @@ def main():
             src.delete_local_zip(zip_file_path)
         elif "mysqldump" in job["job_name"]:
             src.remote_channel_command(job, "mysqldump")
-            zipname = src.remote_channel_command(job, "zip_mysqldump")
+            zipname = src.remote_channel_command(job, "tar_mysqldump")
             if zipname is None:
                 logging.warning(f'failed zipping remote directory at job {job}')
                 sys.exit(1)
@@ -87,7 +87,7 @@ def main():
                 sys.exit(1)
             src.delete_local_zip(local_zip_path_name)
         elif "backup_remote" in job["job_name"]:
-            zipname = src.remote_channel_command(job, "zip_recursive")
+            zipname = src.remote_channel_command(job, "tar_recursive")
             if zipname is None:
                 logging.warning(f'failed zipping remote directory at job {job}')
                 sys.exit(1)

file_mover/remote_local_dir.py

@@ -2,7 +2,7 @@ from pathlib import Path
 import contextlib
 import os
 from datetime import timedelta, datetime
-from zipfile import ZipFile
+import subprocess
 
 class local_dir_remote:
     def __init__(self, config):
@@ -34,13 +34,12 @@ class local_dir_remote:
 
     def zip_local_directory(self, job):
         source_files = self.list_dir(True)
-        zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.zip'
-        with ZipFile(zipname, 'w') as myzip:
-            for files in source_files:
-                try:
-                    myzip.write(files)
-                except Exception:
-                    continue
+        zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.tar.xz'
+        #-T3 = 3 threads, the backups are meant to run in the night at 5 am to not interfere with players.
+        exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar -cJf {zipname} {self.path}"
+        print("exec_command_state: ", exec_command_state)
+        subprocess.run([exec_command_state], shell=True)
+
         return str(Path.cwd()) + '/' +  zipname
 
     def delete_local_demo(self, pathfile):
@@ -51,6 +50,6 @@ class local_dir_remote:
 
     def delete_local_zip(self, zip_file_path):
         with contextlib.suppress(FileNotFoundError):
-            if (str(zip_file_path).endswith('.zip')):
+            if (str(zip_file_path).endswith('.tar.xz')):
                 os.remove(zip_file_path)
-                print('removed zip: ', zip_file_path)
+                print('removed tar xz: ', zip_file_path)

file_mover/remote_sftp.py

@@ -45,16 +45,18 @@ class sftp_remote:
         self.ssh.close()
 
     def remote_channel_command(self, job, command_state):
-        zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.zip'
+        zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.tar.xz'
         self.ssh_connect()
         channel = self.ssh.get_transport().open_session(timeout=120)
-        if 'zip_recursive' == command_state:
-            exec_command_state = f'zip -r {zipname} {self.path}'
+
+        #-T3= 3 threads, use 70% of available ram
+        if 'tar_recursive' == command_state:
+            exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar -cJf {zipname} {self.path}"
         elif 'mysqldump' == command_state:
             #check README for my.cnf regarding how this works
             exec_command_state = f'mysqldump -u root --all-databases > mysqldump.sql'
-        else: #zip_mysqldump
-            exec_command_state = f'zip {zipname} /home/{self.username}/mysqldump.sql'
+        else: #tar_mysqldump
+            exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar -cJf {zipname} /home/{self.username}/mysqldump.sql"
         exec_cmd = f'cd /home/{self.username}/; {exec_command_state}'
         keyboard_interrupt = False
         try: 
@@ -65,13 +67,13 @@ class sftp_remote:
                 if not buf:
                     break
                 #priting the output for sure causes it to run a bit slower
-                print('buffer: ', buf)
+                #print('buffer: ', buf)
             channel.recv_exit_status()
         except KeyboardInterrupt:
             keyboard_interrupt = True
         finally:
             channel.close()
-            if 'zip_mysqldump' == command_state:
+            if 'tar_mysqldump' == command_state:
                 file_path = f'/home/{self.username}/mysqldump.sql'
                 self.connect()
                 self.sftp.remove(file_path)
@@ -85,8 +87,11 @@ class sftp_remote:
     def get_remote_files(self, job, zipname):
         total_attempts = int(self.remote_attempts)
         while total_attempts > 0:
+            print(f'job: {job}')
             self.connect()
             job_path = job["download_dir"]
+            #print(f'/home/{self.username}/{zipname}')
+            #print(f'{job_path}{zipname}')
             self.sftp.get(f'/home/{self.username}/{zipname}', f'{job_path}{zipname}')
             sha256_first = self.digest(f'{job_path}{zipname}')
             os.remove(f'{job_path}{zipname}')
@@ -199,4 +204,3 @@ class sftp_remote:
 
     def delete_local_zip(self, local_zip_path_name):
         os.remove(local_zip_path_name)
-

file_mover/systemctl/backups.timer

@@ -3,7 +3,7 @@ Description=Creates backups thrice a week
 Requires=backups.service
 
 [Timer]
-OnCalendar=Mon,Thu,Sat 10:00
+OnCalendar=Mon,Thu,Sat 04:00
 
 [Install]
 WantedBy=multi-user.target