From 0cf47b8e6cfbb8274a28f1edcfa5bd3dae2639fc Mon Sep 17 00:00:00 2001 From: jenz Date: Thu, 20 Oct 2022 00:37:58 +0200 Subject: [PATCH] initial commit for uploading just maps over a VM to OVH --- map_mover_ovh/README.md | 6 ++ map_mover_ovh/config_maps.json | 31 ++++++ map_mover_ovh/main.py | 65 +++++++++++++ map_mover_ovh/remote_local_dir.py | 18 ++++ map_mover_ovh/remote_sftp.py | 96 +++++++++++++++++++ map_mover_ovh/systemctl/map.conf | 3 + .../systemctl/move_map_files.service | 15 +++ map_mover_ovh/systemctl/move_map_files.timer | 9 ++ 8 files changed, 243 insertions(+) create mode 100644 map_mover_ovh/README.md create mode 100644 map_mover_ovh/config_maps.json create mode 100644 map_mover_ovh/main.py create mode 100644 map_mover_ovh/remote_local_dir.py create mode 100644 map_mover_ovh/remote_sftp.py create mode 100644 map_mover_ovh/systemctl/map.conf create mode 100644 map_mover_ovh/systemctl/move_map_files.service create mode 100644 map_mover_ovh/systemctl/move_map_files.timer diff --git a/map_mover_ovh/README.md b/map_mover_ovh/README.md new file mode 100644 index 00000000..8ccd7acb --- /dev/null +++ b/map_mover_ovh/README.md @@ -0,0 +1,6 @@ +needs pip3 with virtual environment: +pip3 install paramiko + +the python3 files are chown root:root, also chmod 700 so only root can access them. The systemctl file also ensures that stdout and stderror are null. + +Every 2 mins uploaded maps are moved over to ovh. Also enable and start timer and services. diff --git a/map_mover_ovh/config_maps.json b/map_mover_ovh/config_maps.json new file mode 100644 index 00000000..608235d7 --- /dev/null +++ b/map_mover_ovh/config_maps.json @@ -0,0 +1,31 @@ +{ + "remotes":{ + "local_dir_maps_folder":{ + "description": "local machine ze maps folder", + "path": "/home/nonroot/move_maps/maps/", + "remote_type": "local_dir" + }, + "sftp_ovh_ze_maps_dest":{ + "description": "sftp for uploading maps to ovh css ze maps folder", + "hostname": "", + "username": "", + "port": "22", + "path": "/home/gameservers/css_ze/cstrike/maps/", + "remote_type": "sftp" + } + }, + "jobs":[ + { + "job_name": "move_maps_to_css_ze_maps", + "job_description": "moves maps to css_ze server", + "src": "local_dir_maps_folder", + "dest": "sftp_ovh_ze_maps_dest" + } + ], + "settings": { + "sftp": { + "remote_attempts": "5", + "remote_delay": "15" + } + } +} diff --git a/map_mover_ovh/main.py b/map_mover_ovh/main.py new file mode 100644 index 00000000..76ec6d57 --- /dev/null +++ b/map_mover_ovh/main.py @@ -0,0 +1,65 @@ +import paramiko +import sys +import json + +def load_config(config_file): + try: + with open(config_file) as infile: + try: + json_dict = json.load(infile) + #logging.info(json_dict) + return json_dict["remotes"], json_dict["jobs"], json_dict["settings"] + except json.JSONDecodeError as e: + logging.warning('exception caught: ', exc_info = True) + sys.exit(1) + except FileNotFoundError: + logging.warning('exception caught: ', exc_info = True) + sys.exit(1) + +def create_remote(config, settings): + type_r = config["remote_type"] + if type_r == "sftp": + import remote_sftp + remote = remote_sftp.sftp_remote(config, settings) + elif type_r == "local_dir": + import remote_local_dir + remote = remote_local_dir.local_dir_remote(config) + return remote + +def distribute_files(path_list, dest): + for pathfile in path_list: + if not str(pathfile).endswith('.bsp'): + continue + if not dest.put(pathfile, dest.path): + log_msg = ''.join(["failed putting file: ", str(pathfile)]) + print(log_msg) + sys.exit(1) + +def main(): + config_file = 'config_maps.json' + + ip = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + + remotes, jobs, settings = load_config(config_file) + + remotes["sftp_ovh_ze_maps_dest"]["hostname"] = ip + remotes["sftp_ovh_ze_maps_dest"]["username"] = username + + settings["sftp"][ip] = {} + settings["sftp"][ip][username] = {"password": password} + for index, job in enumerate(jobs): + src = create_remote(remotes[job["src"]], settings) + dest = create_remote(remotes[job["dest"]], settings) + + source_files = src.list_dir() + + distribute_files(source_files, dest) + for pathfile in source_files: + src.delete_local_map(pathfile) + +if __name__ == '__main__': + main() + + diff --git a/map_mover_ovh/remote_local_dir.py b/map_mover_ovh/remote_local_dir.py new file mode 100644 index 00000000..9175516a --- /dev/null +++ b/map_mover_ovh/remote_local_dir.py @@ -0,0 +1,18 @@ +from pathlib import Path +import contextlib +import os + +class local_dir_remote: + def __init__(self, config): + self.config = config + self.path = config["path"] + + def list_dir(self): + path = Path(self.path) + glob_pattern = '*' + return list(path.glob(glob_pattern)) + + def delete_local_map(self, pathfile): + with contextlib.suppress(FileNotFoundError): + if str(pathfile).endswith('.bsp'): + os.remove(pathfile) diff --git a/map_mover_ovh/remote_sftp.py b/map_mover_ovh/remote_sftp.py new file mode 100644 index 00000000..68ee8f9c --- /dev/null +++ b/map_mover_ovh/remote_sftp.py @@ -0,0 +1,96 @@ +import paramiko +from paramiko.ssh_exception import SSHException, NoValidConnectionsError +import time +import hashlib +import sys +import os + +class sftp_remote: + def __init__(self, config, settings): + self.config = config + self.path = config['path'] + self.description = config['description'] + self.hostname = config['hostname'] + self.username = config['username'] + self.port = config['port'] + self.remote_type = config['remote_type'] + self.remote_attempts = settings[self.remote_type]['remote_attempts'] + self.remote_delay = settings[self.remote_type]['remote_delay'] + self.remote_error = None + self.password = settings[self.remote_type][self.hostname][self.username]['password'] + self.ssh = None + + def connect(self): + self.transport = paramiko.Transport((self.hostname, int(self.port))) + self.transport.connect(username = self.username, password = self.password) + self.sftp = paramiko.SFTPClient.from_transport(self.transport) + + def disconnect(self): + del (self.sftp) + self.transport.close() + + def __close__(self): + del (self.sftp) + self.transport.close() + + def ssh_connect(self): + self.ssh = paramiko.SSHClient() + #i trust my vm's + self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.ssh.connect(self.hostname, username=self.username, password=self.password) + + def ssh_disconnect(self): + self.ssh.close() + + def subtract_remote_attempts(self, total_attempts): + time.sleep(self.remote_delay) + return total_attempts -1 + + def digest(self, file_path): + hashvalue = hashlib.sha256() + with open(file_path, 'rb') as file_: + while True: + chunk = file_.read(hashvalue.block_size) + if not chunk: + break + hashvalue.update(chunk) + return hashvalue.hexdigest() + + + def put(self, local_path, remote_path, files_bytes = None): + sha256 = self.digest(local_path) #sha value at source + total_attempts = int(self.remote_attempts) + local_path_str = str(local_path) + local_path_get = ''.join([local_path_str[:local_path_str.rindex('/')]]) + try: + self.connect() + #print('remote_path chdir: ', remote_path) + self.sftp.chdir(remote_path) # Test if remote_path exists + except IOError: + self.sftp.mkdir(remote_path) + finally: + self.disconnect() + if not os.path.isdir(local_path_get): + original_umask = os.umask(0) #create local temp folder for redownloaded files + os.makedirs(local_path_get, 755) + os.umask(original_umask) + + filename = local_path_str.split("/")[-1] + remote_path = remote_path + filename + local_path_get = local_path_get + filename + while total_attempts > 0: + self.connect() + self.sftp.put(local_path, remote_path) + self.disconnect() + self.connect() + self.sftp.get(remote_path, local_path_get) + self.disconnect() + local_sha256 = self.digest(local_path_get) + os.remove(local_path_get) + if local_sha256 == sha256: + #print('sha confirmed') + return True + total_attempts = self.subtract_remote_attempts(total_attempts) + if total_attempts == 0: + return False + diff --git a/map_mover_ovh/systemctl/map.conf b/map_mover_ovh/systemctl/map.conf new file mode 100644 index 00000000..86ba4b4f --- /dev/null +++ b/map_mover_ovh/systemctl/map.conf @@ -0,0 +1,3 @@ +ARG1="ip address" +ARG2="username" +ARG3="password" diff --git a/map_mover_ovh/systemctl/move_map_files.service b/map_mover_ovh/systemctl/move_map_files.service new file mode 100644 index 00000000..09763167 --- /dev/null +++ b/map_mover_ovh/systemctl/move_map_files.service @@ -0,0 +1,15 @@ +[Unit] +Description=Move maps to OVH + +[Service] +Environment=PATH=/home/nonroot/move_maps/venv/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games +EnvironmentFile=/etc/systemd/system/map.conf +User=root +Group=root +WorkingDirectory=/home/nonroot/move_maps +ExecStart=/home/nonroot/move_maps/main.py $ARG1 $ARG2 $ARG3 +StandardOutput=null +StandardError=null + +[Install] +WantedBy=default.target diff --git a/map_mover_ovh/systemctl/move_map_files.timer b/map_mover_ovh/systemctl/move_map_files.timer new file mode 100644 index 00000000..ddc4d819 --- /dev/null +++ b/map_mover_ovh/systemctl/move_map_files.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Moves uploaded maps every 2 minute + +[Timer] +OnUnitActiveSec=120s +OnBootSec=120s + +[Install] +WantedBy=multi-user.target