updated directory name, changed configs and systemctl slightly, preparing to make it useable for backups

file_mover/remote_sftp.py

@@ -0,0 +1,104 @@
+import paramiko
+from paramiko.ssh_exception import SSHException, NoValidConnectionsError
+import time
+import sys
+import hashlib
+import stat
+from datetime import datetime, timedelta
+import os
+
+class sftp_remote:
+    def __init__(self, config, settings):
+        self.config = config
+        self.path = config['path']
+        self.description = config['description']
+        self.hostname = config['hostname']
+        self.username = config['username']
+        self.port = config['port']
+        self.remote_type = config['remote_type']
+        self.remote_attempts = settings[self.remote_type]['remote_attempts']
+        self.remote_delay = settings[self.remote_type]['remote_delay']
+        self.remote_error = None
+        self.password = settings[self.remote_type][self.hostname][self.username]['password']
+
+    def connect(self):
+        self.transport = paramiko.Transport((self.hostname, int(self.port)))
+        self.transport.connect(username = self.username, password = self.password)
+        self.sftp = paramiko.SFTPClient.from_transport(self.transport)
+
+    def disconnect(self):
+        del (self.sftp)
+        self.transport.close()
+
+    def __close__(self):
+        del (self.sftp)
+        self.transport.close()
+
+    def subtract_remote_attempts(self, total_attempts):
+        time.sleep(self.remote_delay)
+        return total_attempts -1
+
+    def digest(self, file_path):
+        hashvalue = hashlib.sha256()
+        with open(file_path, 'rb') as file_:
+            while True:
+                chunk = file_.read(hashvalue.block_size)
+                if not chunk:
+                    break
+                hashvalue.update(chunk)
+        return hashvalue.hexdigest()
+
+
+    def put(self, local_path, remote_path, files_bytes = None):
+        sha256 = self.digest(local_path) #sha value at source
+        total_attempts = int(self.remote_attempts)
+        local_path_str = str(local_path)
+        local_path_get = ''.join([local_path_str[:local_path_str.rindex('/')]])
+        local_temp_folder = local_path_get + "/tempfolder/"
+        try:
+            self.connect()
+            #print('remote_path chdir: ', remote_path)
+            self.sftp.chdir(remote_path)  # Test if remote_path exists
+        except IOError:
+            self.sftp.mkdir(remote_path)
+        finally:
+            self.disconnect()
+            if not os.path.isdir(local_temp_folder):
+                original_umask = os.umask(0) #create local temp folder for redownloaded files
+                os.makedirs(local_temp_folder, 755)
+                os.umask(original_umask)
+
+        filename = local_path_str.split("/")[-1]
+        remote_path = remote_path + filename
+        local_temp_folder = local_temp_folder + filename
+        while total_attempts > 0:
+            self.connect()
+            self.sftp.put(local_path, remote_path)
+            self.disconnect()
+            self.connect()
+            self.sftp.get(remote_path, local_temp_folder)
+            self.disconnect()
+            local_sha256 = self.digest(local_temp_folder)
+            os.remove(local_temp_folder)
+            if local_sha256 == sha256:
+                #print('sha confirmed')
+                return True
+            total_attempts = self.subtract_remote_attempts(total_attempts)
+            if total_attempts == 0:
+                return False
+        
+
+    def delete_remote(self, source_files):
+        self.connect()
+        for pathfile in source_files:
+            pathfile = str(pathfile)
+            if not pathfile.endswith('.dem'):
+                continue
+            filename = pathfile.split("/")[-1]
+            pathfile = self.path + filename
+            utime = self.sftp.stat(pathfile).st_mtime
+            last_modified = datetime.fromtimestamp(utime)
+            if (datetime.now() - last_modified) > timedelta(days=30 * 1.5):
+                print('deleting file remotely: ', pathfile)
+                self.sftp.remove(pathfile)
+        self.disconnect()