initial commit for uploading just maps over a VM to OVH

map_mover_ovh/README.md

@@ -0,0 +1,6 @@
+needs pip3 with virtual environment:
+pip3 install paramiko
+
+the python3 files are chown root:root, also chmod 700 so only root can access them. The systemctl file also ensures that stdout and stderror are null. 
+
+Every 2 mins uploaded maps are moved over to ovh. Also enable and start timer and services.

map_mover_ovh/config_maps.json

@@ -0,0 +1,31 @@
+{
+        "remotes":{
+                "local_dir_maps_folder":{
+                        "description": "local machine ze maps folder",
+                        "path": "/home/nonroot/move_maps/maps/",
+                        "remote_type": "local_dir"
+                },
+                "sftp_ovh_ze_maps_dest":{
+                        "description": "sftp for uploading maps to ovh css ze maps folder",
+                        "hostname": "",
+                        "username": "",
+                        "port": "22",
+                        "path": "/home/gameservers/css_ze/cstrike/maps/",
+                        "remote_type": "sftp"
+                }
+        },
+        "jobs":[
+                {
+                "job_name": "move_maps_to_css_ze_maps",
+                "job_description": "moves maps to css_ze server",
+                "src": "local_dir_maps_folder",
+                "dest": "sftp_ovh_ze_maps_dest"
+                }
+        ],
+        "settings": {
+                "sftp": {
+                        "remote_attempts": "5",
+                        "remote_delay": "15"
+                }
+        }
+}

map_mover_ovh/main.py

@@ -0,0 +1,65 @@
+import paramiko
+import sys
+import json
+
+def load_config(config_file):
+    try:
+        with open(config_file) as infile:
+            try:
+                json_dict = json.load(infile)
+                #logging.info(json_dict)
+                return json_dict["remotes"], json_dict["jobs"], json_dict["settings"]
+            except json.JSONDecodeError as e:
+                logging.warning('exception caught: ', exc_info = True)
+                sys.exit(1)
+    except FileNotFoundError:
+        logging.warning('exception caught: ', exc_info = True)
+        sys.exit(1)
+
+def create_remote(config, settings):
+    type_r = config["remote_type"]
+    if type_r == "sftp":
+        import remote_sftp
+        remote = remote_sftp.sftp_remote(config, settings)
+    elif type_r == "local_dir":
+        import remote_local_dir
+        remote = remote_local_dir.local_dir_remote(config)
+    return remote
+
+def distribute_files(path_list, dest):
+    for pathfile in path_list:
+        if not str(pathfile).endswith('.bsp'):
+            continue
+        if not dest.put(pathfile, dest.path):
+            log_msg = ''.join(["failed putting file: ", str(pathfile)])
+            print(log_msg)
+            sys.exit(1)
+
+def main():
+    config_file = 'config_maps.json'
+
+    ip = sys.argv[1]
+    username = sys.argv[2]
+    password = sys.argv[3]
+
+    remotes, jobs, settings = load_config(config_file)
+
+    remotes["sftp_ovh_ze_maps_dest"]["hostname"] = ip
+    remotes["sftp_ovh_ze_maps_dest"]["username"] = username
+
+    settings["sftp"][ip] =  {}
+    settings["sftp"][ip][username] = {"password": password}
+    for index, job in enumerate(jobs):
+        src = create_remote(remotes[job["src"]], settings)
+        dest = create_remote(remotes[job["dest"]], settings)
+
+        source_files = src.list_dir()
+
+        distribute_files(source_files, dest)
+        for pathfile in source_files:
+            src.delete_local_map(pathfile)
+
+if __name__ == '__main__':
+    main()
+
+

map_mover_ovh/remote_local_dir.py

@@ -0,0 +1,18 @@
+from pathlib import Path
+import contextlib
+import os
+
+class local_dir_remote:
+    def __init__(self, config):
+        self.config = config
+        self.path = config["path"]
+
+    def list_dir(self):
+        path = Path(self.path)
+        glob_pattern = '*'
+        return list(path.glob(glob_pattern))
+
+    def delete_local_map(self, pathfile):
+        with contextlib.suppress(FileNotFoundError):
+            if str(pathfile).endswith('.bsp'):
+                os.remove(pathfile)

map_mover_ovh/remote_sftp.py

@@ -0,0 +1,96 @@
+import paramiko
+from paramiko.ssh_exception import SSHException, NoValidConnectionsError
+import time
+import hashlib
+import sys
+import os
+
+class sftp_remote:
+    def __init__(self, config, settings):
+        self.config = config
+        self.path = config['path']
+        self.description = config['description']
+        self.hostname = config['hostname']
+        self.username = config['username']
+        self.port = config['port']
+        self.remote_type = config['remote_type']
+        self.remote_attempts = settings[self.remote_type]['remote_attempts']
+        self.remote_delay = settings[self.remote_type]['remote_delay']
+        self.remote_error = None
+        self.password = settings[self.remote_type][self.hostname][self.username]['password']
+        self.ssh = None
+
+    def connect(self):
+        self.transport = paramiko.Transport((self.hostname, int(self.port)))
+        self.transport.connect(username = self.username, password = self.password)
+        self.sftp = paramiko.SFTPClient.from_transport(self.transport)
+
+    def disconnect(self):
+        del (self.sftp)
+        self.transport.close()
+
+    def __close__(self):
+        del (self.sftp)
+        self.transport.close()
+
+    def ssh_connect(self):
+        self.ssh = paramiko.SSHClient()
+        #i trust my vm's
+        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        self.ssh.connect(self.hostname, username=self.username, password=self.password)
+
+    def ssh_disconnect(self):
+        self.ssh.close()
+
+    def subtract_remote_attempts(self, total_attempts):
+        time.sleep(self.remote_delay)
+        return total_attempts -1
+
+    def digest(self, file_path):
+        hashvalue = hashlib.sha256()
+        with open(file_path, 'rb') as file_:
+            while True:
+                chunk = file_.read(hashvalue.block_size)
+                if not chunk:
+                    break
+                hashvalue.update(chunk)
+        return hashvalue.hexdigest()
+
+
+    def put(self, local_path, remote_path, files_bytes = None):
+        sha256 = self.digest(local_path) #sha value at source
+        total_attempts = int(self.remote_attempts)
+        local_path_str = str(local_path)
+        local_path_get = ''.join([local_path_str[:local_path_str.rindex('/')]])
+        try:
+            self.connect()
+            #print('remote_path chdir: ', remote_path)
+            self.sftp.chdir(remote_path)  # Test if remote_path exists
+        except IOError:
+            self.sftp.mkdir(remote_path)
+        finally:
+            self.disconnect()
+            if not os.path.isdir(local_path_get):
+                original_umask = os.umask(0) #create local temp folder for redownloaded files
+                os.makedirs(local_path_get, 755)
+                os.umask(original_umask)
+
+        filename = local_path_str.split("/")[-1]
+        remote_path = remote_path + filename
+        local_path_get = local_path_get + filename
+        while total_attempts > 0:
+            self.connect()
+            self.sftp.put(local_path, remote_path)
+            self.disconnect()
+            self.connect()
+            self.sftp.get(remote_path, local_path_get)
+            self.disconnect()
+            local_sha256 = self.digest(local_path_get)
+            os.remove(local_path_get)
+            if local_sha256 == sha256:
+                #print('sha confirmed')
+                return True
+            total_attempts = self.subtract_remote_attempts(total_attempts)
+            if total_attempts == 0:
+                return False
+

map_mover_ovh/systemctl/map.conf

@@ -0,0 +1,3 @@
+ARG1="ip address"
+ARG2="username"
+ARG3="password"

map_mover_ovh/systemctl/move_map_files.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Move maps to OVH
+
+[Service]
+Environment=PATH=/home/nonroot/move_maps/venv/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games
+EnvironmentFile=/etc/systemd/system/map.conf
+User=root
+Group=root
+WorkingDirectory=/home/nonroot/move_maps
+ExecStart=/home/nonroot/move_maps/main.py $ARG1 $ARG2 $ARG3
+StandardOutput=null
+StandardError=null
+
+[Install]
+WantedBy=default.target

map_mover_ovh/systemctl/move_map_files.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Moves uploaded maps every 2 minute
+
+[Timer]
+OnUnitActiveSec=120s
+OnBootSec=120s
+
+[Install]
+WantedBy=multi-user.target