updated directory name, changed configs and systemctl slightly, preparing to make it useable for backups

file_mover/README.md

@@ -0,0 +1,23 @@
+#simple file mover by jenz
+
+##virtualenv venv
+
+##source venv/bin/activate
+
+##pip3 install -r requirements.txt
+
+##config file: server specifics
+
+##globally: apt-get install zip
+
+
+##python3 file_mover.py config.json
+
+##systemctl enable file_mover.timer
+
+##systemctl start file_mover.timer
+
+##logging: journalctl -f -u file_mover.service
+
+##logging: systemctl status file_mover.service
+

file_mover/config_backups.json

@@ -0,0 +1,57 @@
+{
+	"remotes":{
+		"local_dir_css_gg":{
+			"description": "local machine gg server",
+                        "path": "/home/gameservers/css_gg/",
+                        "remote_type": "local_dir"
+		},
+		 "local_dir_css_zr":{
+                        "description": "local machine zr server",
+                        "path": "/home/gameservers/css_zr/",
+                        "remote_type": "local_dir"
+                },
+		 "local_dir_css_mg":{
+                        "description": "local machine mg server",
+                        "path": "/home/gameservers/css_gg/",
+                        "remote_type": "local_dir"
+                },
+		 "local_dir_css_ze":{
+                        "description": "local machine ze server",
+                        "path": "/home/gameservers/css_ze/",
+                        "remote_type": "local_dir"
+                },
+		 "local_dir_css_jb":{
+                        "description": "local machine jb server",
+                        "path": "/home/gameservers/css_jb/",
+                        "remote_type": "local_dir"
+                },
+		"sftp_vm_backups_dest":{
+                        "description": "sftp server for backups, fastdl & demos",
+                        "hostname": "163.172.119.53",
+                        "username": "nonroot",
+                        "port": "22",
+                        "path": "/home/nonroot/backups/",
+                        "remote_type": "sftp"
+                }
+	},
+	"jobs":[
+		{
+		"job_name": "backup_local_server_gg",
+		"job_description": "zips the gg server folder and moves it to backup/fastdl/demo vm",
+		"src": "local_dir_css_gg",
+		"dest": "sftp_vm_backups_dest",
+		"zipname":"css_gg_backup"
+		}
+	],
+	"settings": {
+		"sftp": {
+			"remote_attempts": "5",
+			"remote_delay": "15",
+			"163.172.119.53":{
+				"username": {
+					"password": "password123"
+				}
+			}
+		}
+	}
+}

file_mover/file_mover.py

@@ -0,0 +1,68 @@
+import paramiko
+import sys
+import json
+import logging
+import unicodedata
+logging.basicConfig(
+    format='%(asctime)s %(levelname)-8s %(message)s',
+    level=logging.INFO,
+    datefmt='%Y-%m-%d %H:%M:%S'
+        )
+
+def create_remote(config, settings):
+    type_r = config["remote_type"]
+    if type_r == "sftp":
+        import remote_sftp
+        remote = remote_sftp.sftp_remote(config, settings)
+    elif type_r == "local_dir":
+        import remote_local_dir
+        remote = remote_local_dir.local_dir_remote(config)
+    return remote
+
+def distribute_files(path_list, dest):
+    for pathfile in path_list:
+        if not str(pathfile).endswith('.dem'):
+            continue
+        log_msg = "uploading demo: " + str(pathfile)
+        logging.info(log_msg)
+        if not dest.put(pathfile, dest.path):
+            log_msg = ''.join(["failed putting file: ", str(pathfile)])
+            logging.warning(log_msg)
+            sys.exit(1)
+
+def load_config(config_file):
+    try:
+        with open(config_file) as infile:
+            try:
+                json_dict = json.load(infile)
+                #logging.info(json_dict)
+                return json_dict["remotes"], json_dict["jobs"], json_dict["settings"]
+            except json.JSONDecodeError as e:
+                logging.warning('exception caught: ', exc_info = True)
+                sys.exit(1)
+    except FileNotFoundError:
+        logging.warning('exception caught: ', exc_info = True)
+        sys.exit(1)
+
+def main():
+    #runs demos every 5 minutes but backups thrice a week with systemctl services/timers
+    config_file = 'config_demos.json'
+    if sys.argv[1:]:
+        config_file = sys.argv[1]
+    remotes, jobs, settings = load_config(config_file)
+    for job in jobs:
+        src = create_remote(remotes[job["src"]], settings)
+        dest = create_remote(remotes[job["dest"]], settings)
+        if 'demo_' in job["job_name"]:
+            source_files = src.list_dir()
+            distribute_files(source_files, dest)
+            for pathfile in source_files:
+                src.delete_local_demo(pathfile)
+            dest.delete_remote(source_files)
+        elif "backup_local" in job["job_name"]:
+            zip_directory_cmd = f'zip -r {job["zipname"]}-{str(datetime.now()).split(" ")[0]}.zip {remotes[job["src"]]["path"]}'
+
+        logging.info(('finished job: ', job))
+
+if __name__ == '__main__':
+    main()

file_mover/remote_local_dir.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+import contextlib
+import os
+from datetime import timedelta, datetime
+
+class local_dir_remote:
+    def __init__(self, config):
+        self.config = config
+        self.path = config["path"]
+
+    def list_dir(self):
+        path = Path(self.path)
+        file_list = []
+        delta = timedelta(minutes=2)
+        for file_demo in list(path.glob('*')):
+            mtime = datetime.fromtimestamp(os.stat(file_demo).st_mtime)
+            if mtime < datetime.now() - delta:
+                file_list.append(file_demo)
+        return file_list
+
+    def delete_local_demo(self, pathfile):
+        with contextlib.suppress(FileNotFoundError):
+            if str(pathfile).endswith('.dem'):
+                os.remove(pathfile)
+                print('removed demo: ', pathfile)

file_mover/remote_sftp.py

@@ -0,0 +1,104 @@
+import paramiko
+from paramiko.ssh_exception import SSHException, NoValidConnectionsError
+import time
+import sys
+import hashlib
+import stat
+from datetime import datetime, timedelta
+import os
+
+class sftp_remote:
+    def __init__(self, config, settings):
+        self.config = config
+        self.path = config['path']
+        self.description = config['description']
+        self.hostname = config['hostname']
+        self.username = config['username']
+        self.port = config['port']
+        self.remote_type = config['remote_type']
+        self.remote_attempts = settings[self.remote_type]['remote_attempts']
+        self.remote_delay = settings[self.remote_type]['remote_delay']
+        self.remote_error = None
+        self.password = settings[self.remote_type][self.hostname][self.username]['password']
+
+    def connect(self):
+        self.transport = paramiko.Transport((self.hostname, int(self.port)))
+        self.transport.connect(username = self.username, password = self.password)
+        self.sftp = paramiko.SFTPClient.from_transport(self.transport)
+
+    def disconnect(self):
+        del (self.sftp)
+        self.transport.close()
+
+    def __close__(self):
+        del (self.sftp)
+        self.transport.close()
+
+    def subtract_remote_attempts(self, total_attempts):
+        time.sleep(self.remote_delay)
+        return total_attempts -1
+
+    def digest(self, file_path):
+        hashvalue = hashlib.sha256()
+        with open(file_path, 'rb') as file_:
+            while True:
+                chunk = file_.read(hashvalue.block_size)
+                if not chunk:
+                    break
+                hashvalue.update(chunk)
+        return hashvalue.hexdigest()
+
+
+    def put(self, local_path, remote_path, files_bytes = None):
+        sha256 = self.digest(local_path) #sha value at source
+        total_attempts = int(self.remote_attempts)
+        local_path_str = str(local_path)
+        local_path_get = ''.join([local_path_str[:local_path_str.rindex('/')]])
+        local_temp_folder = local_path_get + "/tempfolder/"
+        try:
+            self.connect()
+            #print('remote_path chdir: ', remote_path)
+            self.sftp.chdir(remote_path)  # Test if remote_path exists
+        except IOError:
+            self.sftp.mkdir(remote_path)
+        finally:
+            self.disconnect()
+            if not os.path.isdir(local_temp_folder):
+                original_umask = os.umask(0) #create local temp folder for redownloaded files
+                os.makedirs(local_temp_folder, 755)
+                os.umask(original_umask)
+
+        filename = local_path_str.split("/")[-1]
+        remote_path = remote_path + filename
+        local_temp_folder = local_temp_folder + filename
+        while total_attempts > 0:
+            self.connect()
+            self.sftp.put(local_path, remote_path)
+            self.disconnect()
+            self.connect()
+            self.sftp.get(remote_path, local_temp_folder)
+            self.disconnect()
+            local_sha256 = self.digest(local_temp_folder)
+            os.remove(local_temp_folder)
+            if local_sha256 == sha256:
+                #print('sha confirmed')
+                return True
+            total_attempts = self.subtract_remote_attempts(total_attempts)
+            if total_attempts == 0:
+                return False
+        
+
+    def delete_remote(self, source_files):
+        self.connect()
+        for pathfile in source_files:
+            pathfile = str(pathfile)
+            if not pathfile.endswith('.dem'):
+                continue
+            filename = pathfile.split("/")[-1]
+            pathfile = self.path + filename
+            utime = self.sftp.stat(pathfile).st_mtime
+            last_modified = datetime.fromtimestamp(utime)
+            if (datetime.now() - last_modified) > timedelta(days=30 * 1.5):
+                print('deleting file remotely: ', pathfile)
+                self.sftp.remove(pathfile)
+        self.disconnect()

file_mover/requirements.txt

@@ -0,0 +1 @@
+paramiko

file_mover/run_backups.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+cd $(dirname $0)
+source venv/bin/activate
+python3 file_mover.py config_backups.json

file_mover/run_demo_mover.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+cd $(dirname $0)
+source venv/bin/activate
+python3 file_mover.py config_demos.json

file_mover/systemctl/backups.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Creates backups thrice a week
+
+[Service]
+Type=simple
+User=file_mover
+WorkingDirectory=/home/file_mover
+ExecStart=/home/file_mover/run_backups.sh
+

file_mover/systemctl/backups.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Creates backups thrice a week
+Requires=backups.service
+
+[Timer]
+OnCalendar=Mon,Thu,Sat 10:00
+
+[Install]
+WantedBy=multi-user.target

file_mover/systemctl/demo_mover.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=Moves demos from the local machine to remote destinations every 5 minutes
+
+[Service]
+Type=simple
+User=file_mover
+WorkingDirectory=/home/file_mover
+ExecStart=/home/file_mover/run_demo_mover.sh

file_mover/systemctl/demo_mover.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Moves demos from the local machine to remote destinations every 5 minutes
+Requires=demo_mover.service
+
+[Timer]
+OnCalendar=*-*-* *:0,5,10,15,20,25,30,35,40,45,50,55
+
+[Install]
+WantedBy=multi-user.target