diff --git a/demo_mover/README.md b/demo_mover/README.md deleted file mode 100755 index 50fee556..00000000 --- a/demo_mover/README.md +++ /dev/null @@ -1,22 +0,0 @@ -#simple demo file mover by jenz - -##virtualenv venv - -##source venv/bin/activate - -##pip3 install -r requirements.txt - -##config file: server specifics - - -##python3 demo_mover.py config.json - -##systemctl enable demo_mover.timer - -##systemctl start demo_mover.timer - -##logging: journalctl -f -u demo_mover.service - -##logging: systemctl status demo_mover.service - -###test commit on new git diff --git a/demo_mover/config.json b/demo_mover/config.json deleted file mode 100755 index 41fe8464..00000000 --- a/demo_mover/config.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "remotes":{ - "local_dir_demos_ze_src":{ - "description": "local machine demos ze", - "path": "/home/gameservers/css_ze/cstrike/demos/demos/", - "remote_type": "local_dir" - }, - "local_dir_demos_mg_src":{ - "description": "local machine demos mg", - "path": "/home/gameservers/css_mg/cstrike/demos/", - "remote_type": "local_dir" - }, - "sftp_vm_104_dest":{ - "description": "sftp server for backups, fastdl & demos", - "hostname": "163.172.117.46", - "username": "gameservers", - "port": "22", - "path": "/home/gameservers/demos/", - "remote_type": "sftp" - } - }, - "jobs":[ - { - "job_name": "demo_move_ze", - "job_description": "move demos to vm 104", - "src": "local_dir_demos_ze_src", - "dest": "sftp_vm_104_dest" - }, - { - "job_name": "demo_move_mg", - "job_description": "move demos to vm 104", - "src": "local_dir_demos_mg_src", - "dest": "sftp_vm_104_dest" - } - ], - "settings": { - "sftp": { - "remote_attempts": "5", - "remote_delay": "15", - "163.172.117.46":{ - "gameservers": { - "password": "ndfybf3u24bfapbfpabfib3rIFB" - } - } - } - } -} diff --git a/demo_mover/demo_mover.py b/demo_mover/demo_mover.py deleted file mode 100755 index a9b2fa43..00000000 --- a/demo_mover/demo_mover.py +++ /dev/null @@ -1,63 +0,0 @@ -import paramiko -import sys -import json -import logging -import unicodedata -logging.basicConfig( - format='%(asctime)s %(levelname)-8s %(message)s', - level=logging.INFO, - datefmt='%Y-%m-%d %H:%M:%S' - ) - -def create_remote(config, settings): - type_r = config["remote_type"] - if type_r == "sftp": - import remote_sftp - remote = remote_sftp.sftp_remote(config, settings) - elif type_r == "local_dir": - import remote_local_dir - remote = remote_local_dir.local_dir_remote(config) - return remote - -def distribute_files(path_list, dest): - for pathfile in path_list: - if not str(pathfile).endswith('.dem'): - continue - log_msg = "uploading demo: " + str(pathfile) - logging.info(log_msg) - if not dest.put(pathfile, dest.path): - log_msg = ''.join(["failed putting file: ", str(pathfile)]) - logging.warning(log_msg) - sys.exit(1) - -def load_config(config_file): - try: - with open(config_file) as infile: - try: - json_dict = json.load(infile) - #logging.info(json_dict) - return json_dict["remotes"], json_dict["jobs"], json_dict["settings"] - except json.JSONDecodeError as e: - logging.warning('exception caught: ', exc_info = True) - sys.exit(1) - except FileNotFoundError: - logging.warning('exception caught: ', exc_info = True) - sys.exit(1) - -def main(): - config_file = 'config.json' - if sys.argv[1:]: - config_file = sys.argv[1] - remotes, jobs, settings = load_config(config_file) - for job in jobs: - src = create_remote(remotes[job["src"]], settings) - dest = create_remote(remotes[job["dest"]], settings) - source_files = src.list_dir() - distribute_files(source_files, dest) - for pathfile in source_files: - src.delete_local(pathfile) - dest.delete_remote(source_files) - logging.info(('finished job: ', job)) - -if __name__ == '__main__': - main() diff --git a/demo_mover/remote_local_dir.py b/demo_mover/remote_local_dir.py deleted file mode 100755 index b9dc5acc..00000000 --- a/demo_mover/remote_local_dir.py +++ /dev/null @@ -1,25 +0,0 @@ -from pathlib import Path -import contextlib -import os -from datetime import timedelta, datetime - -class local_dir_remote: - def __init__(self, config): - self.config = config - self.path = config["path"] - - def list_dir(self): - path = Path(self.path) - file_list = [] - delta = timedelta(minutes=2) - for file_demo in list(path.glob('*')): - mtime = datetime.fromtimestamp(os.stat(file_demo).st_mtime) - if mtime < datetime.now() - delta: - file_list.append(file_demo) - return file_list - - def delete_local(self, pathfile): - with contextlib.suppress(FileNotFoundError): - if str(pathfile).endswith('.dem'): - os.remove(pathfile) - print('removed demo: ', pathfile) diff --git a/demo_mover/remote_sftp.py b/demo_mover/remote_sftp.py deleted file mode 100755 index a097fded..00000000 --- a/demo_mover/remote_sftp.py +++ /dev/null @@ -1,104 +0,0 @@ -import paramiko -from paramiko.ssh_exception import SSHException, NoValidConnectionsError -import time -import sys -import hashlib -import stat -from datetime import datetime, timedelta -import os - -class sftp_remote: - def __init__(self, config, settings): - self.config = config - self.path = config['path'] - self.description = config['description'] - self.hostname = config['hostname'] - self.username = config['username'] - self.port = config['port'] - self.remote_type = config['remote_type'] - self.remote_attempts = settings[self.remote_type]['remote_attempts'] - self.remote_delay = settings[self.remote_type]['remote_delay'] - self.remote_error = None - self.password = settings[self.remote_type][self.hostname][self.username]['password'] - - def connect(self): - self.transport = paramiko.Transport((self.hostname, int(self.port))) - self.transport.connect(username = self.username, password = self.password) - self.sftp = paramiko.SFTPClient.from_transport(self.transport) - - def disconnect(self): - del (self.sftp) - self.transport.close() - - def __close__(self): - del (self.sftp) - self.transport.close() - - def subtract_remote_attempts(self, total_attempts): - time.sleep(self.remote_delay) - return total_attempts -1 - - def digest(self, file_path): - hashvalue = hashlib.sha256() - with open(file_path, 'rb') as file_: - while True: - chunk = file_.read(hashvalue.block_size) - if not chunk: - break - hashvalue.update(chunk) - return hashvalue.hexdigest() - - - def put(self, local_path, remote_path, files_bytes = None): - sha256 = self.digest(local_path) #sha value at source - total_attempts = int(self.remote_attempts) - local_path_str = str(local_path) - local_path_get = ''.join([local_path_str[:local_path_str.rindex('/')]]) - local_temp_folder = local_path_get + "/tempfolder/" - try: - self.connect() - #print('remote_path chdir: ', remote_path) - self.sftp.chdir(remote_path) # Test if remote_path exists - except IOError: - self.sftp.mkdir(remote_path) - finally: - self.disconnect() - if not os.path.isdir(local_temp_folder): - original_umask = os.umask(0) #create local temp folder for redownloaded files - os.makedirs(local_temp_folder, 755) - os.umask(original_umask) - - filename = local_path_str.split("/")[-1] - remote_path = remote_path + filename - local_temp_folder = local_temp_folder + filename - while total_attempts > 0: - self.connect() - self.sftp.put(local_path, remote_path) - self.disconnect() - self.connect() - self.sftp.get(remote_path, local_temp_folder) - self.disconnect() - local_sha256 = self.digest(local_temp_folder) - os.remove(local_temp_folder) - if local_sha256 == sha256: - #print('sha confirmed') - return True - total_attempts = self.subtract_remote_attempts(total_attempts) - if total_attempts == 0: - return False - - - def delete_remote(self, source_files): - self.connect() - for pathfile in source_files: - pathfile = str(pathfile) - if not pathfile.endswith('.dem'): - continue - filename = pathfile.split("/")[-1] - pathfile = self.path + filename - utime = self.sftp.stat(pathfile).st_mtime - last_modified = datetime.fromtimestamp(utime) - if (datetime.now() - last_modified) > timedelta(days=30 * 1.5): - print('deleting file remotely: ', pathfile) - self.sftp.remove(pathfile) - self.disconnect() diff --git a/demo_mover/requirements.txt b/demo_mover/requirements.txt deleted file mode 100755 index 8608c1b0..00000000 --- a/demo_mover/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -paramiko diff --git a/demo_mover/run_demo_mover.sh b/demo_mover/run_demo_mover.sh deleted file mode 100755 index 5efe32b8..00000000 --- a/demo_mover/run_demo_mover.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -cd $(dirname $0) -source venv/bin/activate -python3 demo_mover.py config.json diff --git a/demo_mover/systemd/demo_mover.service b/demo_mover/systemd/demo_mover.service deleted file mode 100755 index 07104a65..00000000 --- a/demo_mover/systemd/demo_mover.service +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Moves demos from the local machine to remote destinations every 5 minutes - -[Service] -Type=simple -User=gameservers -WorkingDirectory=/home/gameservers/demo_mover -ExecStart=/home/gameservers/demo_mover/run_demo_mover.sh - diff --git a/demo_mover/systemd/demo_mover.timer b/demo_mover/systemd/demo_mover.timer deleted file mode 100755 index 15de12cd..00000000 --- a/demo_mover/systemd/demo_mover.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Moves demos from the local machine to remote destinations every 5 minutes -Requires=demo_mover.service - -[Timer] -OnCalendar=*-*-* *:0,5,10,15,20,25,30,35,40,45,50,55 - -[Install] -WantedBy=multi-user.target