diff --git a/demo_mover/README.md b/demo_mover/README.md new file mode 100644 index 00000000..dc2def79 --- /dev/null +++ b/demo_mover/README.md @@ -0,0 +1,11 @@ +#simple demo file mover by jenz + +##virtualenv venv + +##source venv/bin/activate + +##pip3 install -r requirements.txt + +##config file: server specifics + +##nohup screen -d -m -S demo_mover python3 demo_mover.py config.json diff --git a/demo_mover/config.json b/demo_mover/config.json new file mode 100644 index 00000000..e3e90daf --- /dev/null +++ b/demo_mover/config.json @@ -0,0 +1,47 @@ +{ + "remotes":{ + "local_dir_demos_ze_src":{ + "description": "local machine demos ze", + "path": "/home/gameservers/css_ze/cstrike/demos/demos/", + "remote_type": "local_dir" + }, + "local_dir_demos_mg_src":{ + "description": "local machine demos mg", + "path": "/home/gameservers/css_mg/cstrike/demos/", + "remote_type": "local_dir" + }, + "sftp_vm_104_dest":{ + "description": "sftp server for backups, fastdl & demos", + "hostname": "163.172.117.46", + "username": "gameservers", + "port": "22", + "path": "/var/www/demos/", + "remote_type": "sftp" + } + }, + "jobs":[ + { + "job_name": "demo_move_ze", + "job_description": "move demos to vm 104", + "src": "local_dir_demos_ze_src", + "dest": "sftp_vm_104_dest" + }, + { + "job_name": "demo_move_mg", + "job_description": "move demos to vm 104", + "src": "local_dir_demos_mg_src", + "dest": "sftp_vm_104_dest" + } + ], + "settings": { + "sftp": { + "remote_attempts": "5", + "remote_delay": "15", + "163.172.117.46":{ + "gameservers": { + "password": "ndfybf3u24bfapbfpabfib3rIFB" + } + } + } + } +} diff --git a/demo_mover/demo_mover.py b/demo_mover/demo_mover.py new file mode 100644 index 00000000..9e43d22f --- /dev/null +++ b/demo_mover/demo_mover.py @@ -0,0 +1,65 @@ +import paramiko +import sys +import json +import logging +import unicodedata +logging.basicConfig( + format='%(asctime)s %(levelname)-8s %(message)s', + level=logging.INFO, + datefmt='%Y-%m-%d %H:%M:%S' + ) + +def create_remote(config, settings): + type_r = config["remote_type"] + if type_r == "sftp": + import remote_sftp + remote = remote_sftp.sftp_remote(config, settings) + elif type_r == "local_dir": + import remote_local_dir + remote = remote_local_dir.local_dir_remote(config) + return remote + +def distribute_files(path_list, dest): + for pathfile in path_list: + log_msg = "uploading demo: " + str(pathfile) + logging.info(log_msg) + if not dest.put(pathfile, dest.path): + log_msg = ''.join(["failed putting file: ", str(pathfile)]) + logging.warning(log_msg) + sys.exit(1) + +def load_config(config_file): + try: + with open(config_file) as infile: + try: + json_dict = json.load(infile) + #logging.info(json_dict) + return json_dict["remotes"], json_dict["jobs"], json_dict["settings"] + except json.JSONDecodeError as e: + logging.warning('exception caught: ', exc_info = True) + sys.exit(1) + except FileNotFoundError: + logging.warning('exception caught: ', exc_info = True) + sys.exit(1) + +def main(): + config_file = 'config.json' + if sys.argv[1:]: + config_file = sys.argv[1] + remotes, jobs, settings = load_config(config_file) + for job in jobs: + try: + src = create_remote(remotes[job["src"]], settings) + dest = create_remote(remotes[job["dest"]], settings) + source_files = src.list_dir() + distribute_files(source_files, dest) + except Exception as e: + logging.warning('exception caught: ', exc_info = True) + finally: + for pathfile in source_files: + src.delete_local(pathfile) + dest.delete_remote(source_files) + logging.info('finished jobs') + +if __name__ == '__main__': + main() diff --git a/demo_mover/remote_local_dir.py b/demo_mover/remote_local_dir.py new file mode 100644 index 00000000..c674125d --- /dev/null +++ b/demo_mover/remote_local_dir.py @@ -0,0 +1,17 @@ +from pathlib import Path +import contextlib +import os + +class local_dir_remote: + def __init__(self, config): + self.config = config + self.path = config["path"] + + def list_dir(self): + path = Path(self.path) + return list(path.glob('*')) + + def delete_local(self, pathfile): + with contextlib.suppress(FileNotFoundError): + os.remove(pathfile) + print('removed demo: ', pathfile) diff --git a/demo_mover/remote_sftp.py b/demo_mover/remote_sftp.py new file mode 100644 index 00000000..cc59401c --- /dev/null +++ b/demo_mover/remote_sftp.py @@ -0,0 +1,119 @@ +import paramiko +from paramiko.ssh_exception import SSHException, NoValidConnectionsError +import time +import sys +import hashlib +import stat +from datetime import datetime, timedelta +import os + +class sftp_remote: + def __init__(self, config, settings): + self.config = config + self.path = config['path'] + self.description = config['description'] + self.hostname = config['hostname'] + self.username = config['username'] + self.port = config['port'] + self.remote_type = config['remote_type'] + self.remote_attempts = settings[self.remote_type]['remote_attempts'] + self.remote_delay = settings[self.remote_type]['remote_delay'] + self.remote_error = None + self.password = settings[self.remote_type][self.hostname][self.username]['password'] + + def connect(self): + total_attempts = int(self.remote_attempts) + self.transport = paramiko.Transport((self.hostname, int(self.port))) + while total_attempts > 0: + try: + self.transport.connect(username = self.username, password = self.password) + self.sftp = paramiko.SFTPClient.from_transport(self.transport) + return True + except NoValidConnectionsError as e: + print('sftp connect failed: ', e) + total_attempts = self.subtract_remote_attempts(total_attempts) + if total_attempts == 0: + self.remote_error = e + return False + + def disconnect(self): + del (self.sftp) + self.transport.close() + + def __close__(self): + del (self.sftp) + self.transport.close() + + def subtract_remote_attempts(self, total_attempts): + time.sleep(self.remote_delay) + return total_attempts -1 + + def digest(self, file_path): + hashvalue = hashlib.sha256() + with open(file_path, 'rb') as file_: + while True: + chunk = file_.read(hashvalue.block_size) + if not chunk: + break + hashvalue.update(chunk) + return hashvalue.hexdigest() + + + def put(self, local_path, remote_path, files_bytes = None): + sha256 = self.digest(local_path) #sha value at source + total_attempts = int(self.remote_attempts) + local_path_str = str(local_path) + local_path_get = ''.join([local_path_str[:local_path_str.rindex('/')]]) + local_temp_folder = local_path_get + "/tempfolder/" + try: + self.connect() + #print('remote_path chdir: ', remote_path) + self.sftp.chdir(remote_path) # Test if remote_path exists + except IOError: + self.sftp.mkdir(remote_path) + finally: + self.disconnect() + if not os.path.isdir(local_temp_folder): + original_umask = os.umask(0) #create local temp folder for redownloaded files + os.makedirs(local_temp_folder, 755) + os.umask(original_umask) + + filename = local_path_str.split("/")[-1] + remote_path = remote_path + filename + local_temp_folder = local_temp_folder + filename + #print('local_path_str: ', local_path_str, '\n remote_path: ', remote_path, '\n local_temp_folder: ', local_temp_folder) + + while total_attempts > 0: + try: + self.connect() + self.sftp.put(local_path, remote_path) + self.disconnect() + self.connect() + self.sftp.get(remote_path, local_temp_folder) + self.disconnect() + local_sha256 = self.digest(local_temp_folder) + os.remove(local_temp_folder) + #print('removing: ', local_temp_folder) + if local_sha256 == sha256: + #print('sha confirmed') + return True + except SSHException as e: + print(e) + self.remote_error = e + total_attempts = self.subtract_remote_attempts(total_attempts) + if total_attempts == 0: + return False + + + def delete_remote(self, source_files): + self.connect() + for pathfile in source_files: + pathfile = str(pathfile) + filename = pathfile.split("/")[-1] + pathfile = self.path + filename + utime = self.sftp.stat(pathfile).st_mtime + last_modified = datetime.fromtimestamp(utime) + if (datetime.now() - last_modified) > timedelta(days=30 * 3): + print('deleting file remotely: ', pathfile) + self.sftp.remove(path) + self.disconnect() diff --git a/demo_mover/requirements.txt b/demo_mover/requirements.txt new file mode 100644 index 00000000..8608c1b0 --- /dev/null +++ b/demo_mover/requirements.txt @@ -0,0 +1 @@ +paramiko