initial commit of demo file mover

demo_mover/README.md

@@ -0,0 +1,11 @@
+#simple demo file mover by jenz
+
+##virtualenv venv
+
+##source venv/bin/activate
+
+##pip3 install -r requirements.txt
+
+##config file: server specifics
+
+##nohup screen -d -m -S demo_mover python3 demo_mover.py config.json

demo_mover/config.json

@@ -0,0 +1,47 @@
+{
+	"remotes":{
+		"local_dir_demos_ze_src":{
+			"description": "local machine demos ze",
+			"path": "/home/gameservers/css_ze/cstrike/demos/demos/",
+			"remote_type": "local_dir"
+		},
+		"local_dir_demos_mg_src":{
+			"description": "local machine demos mg",
+			"path": "/home/gameservers/css_mg/cstrike/demos/",
+			"remote_type": "local_dir"
+		},
+		"sftp_vm_104_dest":{
+			"description": "sftp server for backups, fastdl & demos",
+			"hostname": "163.172.117.46",
+			"username": "gameservers",
+			"port": "22",
+			"path": "/var/www/demos/",
+			"remote_type": "sftp"
+		}
+	},
+	"jobs":[
+		{
+		"job_name": "demo_move_ze",
+		"job_description": "move demos to vm 104",
+		"src": "local_dir_demos_ze_src",
+		"dest": "sftp_vm_104_dest"
+		},
+		{
+		"job_name": "demo_move_mg",
+		"job_description": "move demos to vm 104",
+		"src": "local_dir_demos_mg_src",
+		"dest": "sftp_vm_104_dest"
+		}
+	],
+	"settings": {
+		"sftp": {
+			"remote_attempts": "5",
+			"remote_delay": "15",
+			"163.172.117.46":{
+				"gameservers": {
+					"password": "ndfybf3u24bfapbfpabfib3rIFB"
+				}
+			}
+		}
+	}
+}

demo_mover/demo_mover.py

@@ -0,0 +1,65 @@
+import paramiko
+import sys
+import json
+import logging
+import unicodedata
+logging.basicConfig(
+    format='%(asctime)s %(levelname)-8s %(message)s',
+    level=logging.INFO,
+    datefmt='%Y-%m-%d %H:%M:%S'
+        )
+
+def create_remote(config, settings):
+    type_r = config["remote_type"]
+    if type_r == "sftp":
+        import remote_sftp
+        remote = remote_sftp.sftp_remote(config, settings)
+    elif type_r == "local_dir":
+        import remote_local_dir
+        remote = remote_local_dir.local_dir_remote(config)
+    return remote
+
+def distribute_files(path_list, dest):
+    for pathfile in path_list:
+        log_msg = "uploading demo: " + str(pathfile)
+        logging.info(log_msg)
+        if not dest.put(pathfile, dest.path):
+            log_msg = ''.join(["failed putting file: ", str(pathfile)])
+            logging.warning(log_msg)
+            sys.exit(1)
+
+def load_config(config_file):
+    try:
+        with open(config_file) as infile:
+            try:
+                json_dict = json.load(infile)
+                #logging.info(json_dict)
+                return json_dict["remotes"], json_dict["jobs"], json_dict["settings"]
+            except json.JSONDecodeError as e:
+                logging.warning('exception caught: ', exc_info = True)
+                sys.exit(1)
+    except FileNotFoundError:
+        logging.warning('exception caught: ', exc_info = True)
+        sys.exit(1)
+
+def main():
+    config_file = 'config.json'
+    if sys.argv[1:]:
+        config_file = sys.argv[1]
+    remotes, jobs, settings = load_config(config_file)
+    for job in jobs:
+        try:
+            src = create_remote(remotes[job["src"]], settings)
+            dest = create_remote(remotes[job["dest"]], settings)
+            source_files = src.list_dir()
+            distribute_files(source_files, dest)
+        except Exception as e:
+            logging.warning('exception caught: ', exc_info = True)
+        finally:
+            for pathfile in source_files:
+                src.delete_local(pathfile)
+            dest.delete_remote(source_files)
+        logging.info('finished jobs')
+
+if __name__ == '__main__':
+    main()

demo_mover/remote_local_dir.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+import contextlib
+import os
+
+class local_dir_remote:
+    def __init__(self, config):
+        self.config = config
+        self.path = config["path"]
+
+    def list_dir(self):
+        path = Path(self.path)
+        return list(path.glob('*'))
+
+    def delete_local(self, pathfile):
+        with contextlib.suppress(FileNotFoundError):
+            os.remove(pathfile)
+            print('removed demo: ', pathfile)

demo_mover/remote_sftp.py

@@ -0,0 +1,119 @@
+import paramiko
+from paramiko.ssh_exception import SSHException, NoValidConnectionsError
+import time
+import sys
+import hashlib
+import stat
+from datetime import datetime, timedelta
+import os
+
+class sftp_remote:
+    def __init__(self, config, settings):
+        self.config = config
+        self.path = config['path']
+        self.description = config['description']
+        self.hostname = config['hostname']
+        self.username = config['username']
+        self.port = config['port']
+        self.remote_type = config['remote_type']
+        self.remote_attempts = settings[self.remote_type]['remote_attempts']
+        self.remote_delay = settings[self.remote_type]['remote_delay']
+        self.remote_error = None
+        self.password = settings[self.remote_type][self.hostname][self.username]['password']
+
+    def connect(self):
+        total_attempts = int(self.remote_attempts)
+        self.transport = paramiko.Transport((self.hostname, int(self.port)))
+        while total_attempts > 0:
+            try:
+                self.transport.connect(username = self.username, password = self.password)
+                self.sftp = paramiko.SFTPClient.from_transport(self.transport)
+                return True
+            except NoValidConnectionsError as e:
+                print('sftp connect failed: ', e)
+                total_attempts = self.subtract_remote_attempts(total_attempts)
+                if total_attempts == 0:
+                    self.remote_error = e
+                    return False
+
+    def disconnect(self):
+        del (self.sftp)
+        self.transport.close()
+
+    def __close__(self):
+        del (self.sftp)
+        self.transport.close()
+
+    def subtract_remote_attempts(self, total_attempts):
+        time.sleep(self.remote_delay)
+        return total_attempts -1
+
+    def digest(self, file_path):
+        hashvalue = hashlib.sha256()
+        with open(file_path, 'rb') as file_:
+            while True:
+                chunk = file_.read(hashvalue.block_size)
+                if not chunk:
+                    break
+                hashvalue.update(chunk)
+        return hashvalue.hexdigest()
+
+
+    def put(self, local_path, remote_path, files_bytes = None):
+        sha256 = self.digest(local_path) #sha value at source
+        total_attempts = int(self.remote_attempts)
+        local_path_str = str(local_path)
+        local_path_get = ''.join([local_path_str[:local_path_str.rindex('/')]])
+        local_temp_folder = local_path_get + "/tempfolder/"
+        try:
+            self.connect()
+            #print('remote_path chdir: ', remote_path)
+            self.sftp.chdir(remote_path)  # Test if remote_path exists
+        except IOError:
+            self.sftp.mkdir(remote_path)
+        finally:
+            self.disconnect()
+            if not os.path.isdir(local_temp_folder):
+                original_umask = os.umask(0) #create local temp folder for redownloaded files
+                os.makedirs(local_temp_folder, 755)
+                os.umask(original_umask)
+
+        filename = local_path_str.split("/")[-1]
+        remote_path = remote_path + filename
+        local_temp_folder = local_temp_folder + filename
+        #print('local_path_str: ', local_path_str, '\n remote_path: ', remote_path, '\n local_temp_folder: ', local_temp_folder)
+        
+        while total_attempts > 0:
+            try:
+                self.connect()
+                self.sftp.put(local_path, remote_path)
+                self.disconnect()
+                self.connect()
+                self.sftp.get(remote_path, local_temp_folder)
+                self.disconnect()
+                local_sha256 = self.digest(local_temp_folder)
+                os.remove(local_temp_folder)
+                #print('removing: ', local_temp_folder)
+                if local_sha256 == sha256:
+                    #print('sha confirmed')
+                    return True
+            except SSHException as e:
+                print(e)
+                self.remote_error = e
+            total_attempts = self.subtract_remote_attempts(total_attempts)
+            if total_attempts == 0:
+                return False
+        
+
+    def delete_remote(self, source_files):
+        self.connect()
+        for pathfile in source_files:
+            pathfile = str(pathfile)
+            filename = pathfile.split("/")[-1]
+            pathfile = self.path + filename
+            utime = self.sftp.stat(pathfile).st_mtime
+            last_modified = datetime.fromtimestamp(utime)
+            if (datetime.now() - last_modified) > timedelta(days=30 * 3):
+                print('deleting file remotely: ', pathfile)
+                self.sftp.remove(path)
+        self.disconnect()

demo_mover/requirements.txt

@@ -0,0 +1 @@
+paramiko