added support for remote jobs to zip /etc/, /var/ and home directories. just missing mysqldumps now

file_mover/remote_sftp.py

@@ -1,7 +1,6 @@
 import paramiko
 from paramiko.ssh_exception import SSHException, NoValidConnectionsError
 import time
-import sys
 import hashlib
 import stat
 from datetime import datetime, timedelta
@@ -20,6 +19,7 @@ class sftp_remote:
         self.remote_delay = settings[self.remote_type]['remote_delay']
         self.remote_error = None
         self.password = settings[self.remote_type][self.hostname][self.username]['password']
+        self.ssh = None
 
     def connect(self):
         self.transport = paramiko.Transport((self.hostname, int(self.port)))
@@ -34,6 +34,64 @@ class sftp_remote:
         del (self.sftp)
         self.transport.close()
 
+    def ssh_connect(self):
+        self.ssh = paramiko.SSHClient()
+        #i trust my vm's
+        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        self.ssh.connect(self.hostname, username=self.username, password=self.password)
+
+    def ssh_disconnect(self):
+        self.ssh.close()
+
+    def zip_remote_directory(self, job):
+        zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.zip'
+        self.ssh_connect()
+        channel = self.ssh.get_transport().open_session(timeout=120)
+        exec_cmd = f'cd /home/{self.username}/; zip -r {zipname} {self.path}'
+        keyboard_interrupt = False
+        try: 
+            channel.exec_command(exec_cmd)
+            #2^21 (2,097,152) characters before filling up the buffers of exec_command
+            while True:
+                buf = channel.recv(1024)
+                if not buf:
+                    break
+                #priting the output for sure causes it to run a bit slower
+                print('buffer: ', buf)
+            channel.recv_exit_status()
+        except KeyboardInterrupt:
+            keyboard_interrupt = True
+        finally:
+            channel.close()
+            self.ssh_disconnect()
+            if keyboard_interrupt:
+                print("manually interrupted")
+                sys.exit(1)
+        return zipname
+
+    def get_remote_files(self, job, zipname):
+        total_attempts = int(self.remote_attempts)
+        while total_attempts > 0:
+            self.connect()
+            job_path = job["download_dir"]
+            self.sftp.get(f'/home/{self.username}/{zipname}', f'{job_path}{zipname}')
+            sha256_first = self.digest(f'{job_path}{zipname}')
+            os.remove(f'{job_path}{zipname}')
+            self.disconnect()
+
+            #redownloading to validate the SHA sum
+            self.connect()
+            self.sftp.get(f'/home/{self.username}/{zipname}', f'{job_path}{zipname}')
+            sha256_second = self.digest(f'{job_path}{zipname}')
+            self.disconnect()
+            if sha256_first == sha256_second:
+                return f'{job_path}{zipname}'
+            os.remove(f'{job_path}{zipname}')
+            total_attempts = self.subtract_remote_attempts(total_attempts)
+            if total_attempts == 0:
+                return None
+
+    
     def subtract_remote_attempts(self, total_attempts):
         time.sleep(self.remote_delay)
         return total_attempts -1
@@ -115,3 +173,13 @@ class sftp_remote:
             pathfile, last_modified = self.change_path(zip_file_path)
             self.delete_file(last_modified, pathfile, 10)
         self.disconnect()
+
+    def delete_remote_zip_temp(self, zipname):
+        self.connect()
+        self.sftp.remove(f'/home/{self.username}/{zipname}')
+        self.disconnect()
+
+    def delete_local_zip(self, local_zip_path_name):
+        os.remove(local_zip_path_name)
+
+