updated to xz compression

file_mover/remote_sftp.py

@@ -45,16 +45,18 @@ class sftp_remote:
         self.ssh.close()
 
     def remote_channel_command(self, job, command_state):
-        zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.zip'
+        zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.tar.xz'
         self.ssh_connect()
         channel = self.ssh.get_transport().open_session(timeout=120)
-        if 'zip_recursive' == command_state:
-            exec_command_state = f'zip -r {zipname} {self.path}'
+
+        #-T3= 3 threads, use 70% of available ram
+        if 'tar_recursive' == command_state:
+            exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar -cJf {zipname} {self.path}"
         elif 'mysqldump' == command_state:
             #check README for my.cnf regarding how this works
             exec_command_state = f'mysqldump -u root --all-databases > mysqldump.sql'
-        else: #zip_mysqldump
-            exec_command_state = f'zip {zipname} /home/{self.username}/mysqldump.sql'
+        else: #tar_mysqldump
+            exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar -cJf {zipname} /home/{self.username}/mysqldump.sql"
         exec_cmd = f'cd /home/{self.username}/; {exec_command_state}'
         keyboard_interrupt = False
         try: 
@@ -65,13 +67,13 @@ class sftp_remote:
                 if not buf:
                     break
                 #priting the output for sure causes it to run a bit slower
-                print('buffer: ', buf)
+                #print('buffer: ', buf)
             channel.recv_exit_status()
         except KeyboardInterrupt:
             keyboard_interrupt = True
         finally:
             channel.close()
-            if 'zip_mysqldump' == command_state:
+            if 'tar_mysqldump' == command_state:
                 file_path = f'/home/{self.username}/mysqldump.sql'
                 self.connect()
                 self.sftp.remove(file_path)
@@ -85,8 +87,11 @@ class sftp_remote:
     def get_remote_files(self, job, zipname):
         total_attempts = int(self.remote_attempts)
         while total_attempts > 0:
+            print(f'job: {job}')
             self.connect()
             job_path = job["download_dir"]
+            #print(f'/home/{self.username}/{zipname}')
+            #print(f'{job_path}{zipname}')
             self.sftp.get(f'/home/{self.username}/{zipname}', f'{job_path}{zipname}')
             sha256_first = self.digest(f'{job_path}{zipname}')
             os.remove(f'{job_path}{zipname}')
@@ -199,4 +204,3 @@ class sftp_remote:
 
     def delete_local_zip(self, local_zip_path_name):
         os.remove(local_zip_path_name)
-