diff --git a/file_mover/README.md b/file_mover/README.md new file mode 100755 index 00000000..306ce64f --- /dev/null +++ b/file_mover/README.md @@ -0,0 +1,23 @@ +#simple file mover by jenz + +##virtualenv venv + +##source venv/bin/activate + +##pip3 install -r requirements.txt + +##config file: server specifics + +##globally: apt-get install zip + + +##python3 file_mover.py config.json + +##systemctl enable file_mover.timer + +##systemctl start file_mover.timer + +##logging: journalctl -f -u file_mover.service + +##logging: systemctl status file_mover.service + diff --git a/file_mover/config_backups.json b/file_mover/config_backups.json new file mode 100755 index 00000000..f8de0eb2 --- /dev/null +++ b/file_mover/config_backups.json @@ -0,0 +1,57 @@ +{ + "remotes":{ + "local_dir_css_gg":{ + "description": "local machine gg server", + "path": "/home/gameservers/css_gg/", + "remote_type": "local_dir" + }, + "local_dir_css_zr":{ + "description": "local machine zr server", + "path": "/home/gameservers/css_zr/", + "remote_type": "local_dir" + }, + "local_dir_css_mg":{ + "description": "local machine mg server", + "path": "/home/gameservers/css_gg/", + "remote_type": "local_dir" + }, + "local_dir_css_ze":{ + "description": "local machine ze server", + "path": "/home/gameservers/css_ze/", + "remote_type": "local_dir" + }, + "local_dir_css_jb":{ + "description": "local machine jb server", + "path": "/home/gameservers/css_jb/", + "remote_type": "local_dir" + }, + "sftp_vm_backups_dest":{ + "description": "sftp server for backups, fastdl & demos", + "hostname": "163.172.119.53", + "username": "nonroot", + "port": "22", + "path": "/home/nonroot/backups/", + "remote_type": "sftp" + } + }, + "jobs":[ + { + "job_name": "backup_local_server_gg", + "job_description": "zips the gg server folder and moves it to backup/fastdl/demo vm", + "src": "local_dir_css_gg", + "dest": "sftp_vm_backups_dest", + "zipname":"css_gg_backup" + } + ], + "settings": { + "sftp": { + "remote_attempts": "5", + "remote_delay": "15", + "163.172.119.53":{ + "username": { + "password": "password123" + } + } + } + } +} diff --git a/file_mover/file_mover.py b/file_mover/file_mover.py new file mode 100755 index 00000000..10481d32 --- /dev/null +++ b/file_mover/file_mover.py @@ -0,0 +1,68 @@ +import paramiko +import sys +import json +import logging +import unicodedata +logging.basicConfig( + format='%(asctime)s %(levelname)-8s %(message)s', + level=logging.INFO, + datefmt='%Y-%m-%d %H:%M:%S' + ) + +def create_remote(config, settings): + type_r = config["remote_type"] + if type_r == "sftp": + import remote_sftp + remote = remote_sftp.sftp_remote(config, settings) + elif type_r == "local_dir": + import remote_local_dir + remote = remote_local_dir.local_dir_remote(config) + return remote + +def distribute_files(path_list, dest): + for pathfile in path_list: + if not str(pathfile).endswith('.dem'): + continue + log_msg = "uploading demo: " + str(pathfile) + logging.info(log_msg) + if not dest.put(pathfile, dest.path): + log_msg = ''.join(["failed putting file: ", str(pathfile)]) + logging.warning(log_msg) + sys.exit(1) + +def load_config(config_file): + try: + with open(config_file) as infile: + try: + json_dict = json.load(infile) + #logging.info(json_dict) + return json_dict["remotes"], json_dict["jobs"], json_dict["settings"] + except json.JSONDecodeError as e: + logging.warning('exception caught: ', exc_info = True) + sys.exit(1) + except FileNotFoundError: + logging.warning('exception caught: ', exc_info = True) + sys.exit(1) + +def main(): + #runs demos every 5 minutes but backups thrice a week with systemctl services/timers + config_file = 'config_demos.json' + if sys.argv[1:]: + config_file = sys.argv[1] + remotes, jobs, settings = load_config(config_file) + for job in jobs: + src = create_remote(remotes[job["src"]], settings) + dest = create_remote(remotes[job["dest"]], settings) + if 'demo_' in job["job_name"]: + source_files = src.list_dir() + distribute_files(source_files, dest) + for pathfile in source_files: + src.delete_local_demo(pathfile) + dest.delete_remote(source_files) + elif "backup_local" in job["job_name"]: + zip_directory_cmd = f'zip -r {job["zipname"]}-{str(datetime.now()).split(" ")[0]}.zip {remotes[job["src"]]["path"]}' + + logging.info(('finished job: ', job)) + +if __name__ == '__main__': + main() diff --git a/file_mover/remote_local_dir.py b/file_mover/remote_local_dir.py new file mode 100755 index 00000000..de132911 --- /dev/null +++ b/file_mover/remote_local_dir.py @@ -0,0 +1,25 @@ +from pathlib import Path +import contextlib +import os +from datetime import timedelta, datetime + +class local_dir_remote: + def __init__(self, config): + self.config = config + self.path = config["path"] + + def list_dir(self): + path = Path(self.path) + file_list = [] + delta = timedelta(minutes=2) + for file_demo in list(path.glob('*')): + mtime = datetime.fromtimestamp(os.stat(file_demo).st_mtime) + if mtime < datetime.now() - delta: + file_list.append(file_demo) + return file_list + + def delete_local_demo(self, pathfile): + with contextlib.suppress(FileNotFoundError): + if str(pathfile).endswith('.dem'): + os.remove(pathfile) + print('removed demo: ', pathfile) diff --git a/file_mover/remote_sftp.py b/file_mover/remote_sftp.py new file mode 100755 index 00000000..a097fded --- /dev/null +++ b/file_mover/remote_sftp.py @@ -0,0 +1,104 @@ +import paramiko +from paramiko.ssh_exception import SSHException, NoValidConnectionsError +import time +import sys +import hashlib +import stat +from datetime import datetime, timedelta +import os + +class sftp_remote: + def __init__(self, config, settings): + self.config = config + self.path = config['path'] + self.description = config['description'] + self.hostname = config['hostname'] + self.username = config['username'] + self.port = config['port'] + self.remote_type = config['remote_type'] + self.remote_attempts = settings[self.remote_type]['remote_attempts'] + self.remote_delay = settings[self.remote_type]['remote_delay'] + self.remote_error = None + self.password = settings[self.remote_type][self.hostname][self.username]['password'] + + def connect(self): + self.transport = paramiko.Transport((self.hostname, int(self.port))) + self.transport.connect(username = self.username, password = self.password) + self.sftp = paramiko.SFTPClient.from_transport(self.transport) + + def disconnect(self): + del (self.sftp) + self.transport.close() + + def __close__(self): + del (self.sftp) + self.transport.close() + + def subtract_remote_attempts(self, total_attempts): + time.sleep(self.remote_delay) + return total_attempts -1 + + def digest(self, file_path): + hashvalue = hashlib.sha256() + with open(file_path, 'rb') as file_: + while True: + chunk = file_.read(hashvalue.block_size) + if not chunk: + break + hashvalue.update(chunk) + return hashvalue.hexdigest() + + + def put(self, local_path, remote_path, files_bytes = None): + sha256 = self.digest(local_path) #sha value at source + total_attempts = int(self.remote_attempts) + local_path_str = str(local_path) + local_path_get = ''.join([local_path_str[:local_path_str.rindex('/')]]) + local_temp_folder = local_path_get + "/tempfolder/" + try: + self.connect() + #print('remote_path chdir: ', remote_path) + self.sftp.chdir(remote_path) # Test if remote_path exists + except IOError: + self.sftp.mkdir(remote_path) + finally: + self.disconnect() + if not os.path.isdir(local_temp_folder): + original_umask = os.umask(0) #create local temp folder for redownloaded files + os.makedirs(local_temp_folder, 755) + os.umask(original_umask) + + filename = local_path_str.split("/")[-1] + remote_path = remote_path + filename + local_temp_folder = local_temp_folder + filename + while total_attempts > 0: + self.connect() + self.sftp.put(local_path, remote_path) + self.disconnect() + self.connect() + self.sftp.get(remote_path, local_temp_folder) + self.disconnect() + local_sha256 = self.digest(local_temp_folder) + os.remove(local_temp_folder) + if local_sha256 == sha256: + #print('sha confirmed') + return True + total_attempts = self.subtract_remote_attempts(total_attempts) + if total_attempts == 0: + return False + + + def delete_remote(self, source_files): + self.connect() + for pathfile in source_files: + pathfile = str(pathfile) + if not pathfile.endswith('.dem'): + continue + filename = pathfile.split("/")[-1] + pathfile = self.path + filename + utime = self.sftp.stat(pathfile).st_mtime + last_modified = datetime.fromtimestamp(utime) + if (datetime.now() - last_modified) > timedelta(days=30 * 1.5): + print('deleting file remotely: ', pathfile) + self.sftp.remove(pathfile) + self.disconnect() diff --git a/file_mover/requirements.txt b/file_mover/requirements.txt new file mode 100755 index 00000000..8608c1b0 --- /dev/null +++ b/file_mover/requirements.txt @@ -0,0 +1 @@ +paramiko diff --git a/file_mover/run_backups.sh b/file_mover/run_backups.sh new file mode 100755 index 00000000..0ac18f44 --- /dev/null +++ b/file_mover/run_backups.sh @@ -0,0 +1,4 @@ +#!/bin/bash +cd $(dirname $0) +source venv/bin/activate +python3 file_mover.py config_backups.json diff --git a/file_mover/run_demo_mover.sh b/file_mover/run_demo_mover.sh new file mode 100755 index 00000000..a4a2d9d4 --- /dev/null +++ b/file_mover/run_demo_mover.sh @@ -0,0 +1,4 @@ +#!/bin/bash +cd $(dirname $0) +source venv/bin/activate +python3 file_mover.py config_demos.json diff --git a/file_mover/systemctl/backups.service b/file_mover/systemctl/backups.service new file mode 100644 index 00000000..1700b90f --- /dev/null +++ b/file_mover/systemctl/backups.service @@ -0,0 +1,9 @@ +[Unit] +Description=Creates backups thrice a week + +[Service] +Type=simple +User=file_mover +WorkingDirectory=/home/file_mover +ExecStart=/home/file_mover/run_backups.sh + diff --git a/file_mover/systemctl/backups.timer b/file_mover/systemctl/backups.timer new file mode 100644 index 00000000..7f6e77e6 --- /dev/null +++ b/file_mover/systemctl/backups.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Creates backups thrice a week +Requires=backups.service + +[Timer] +OnCalendar=Mon,Thu,Sat 10:00 + +[Install] +WantedBy=multi-user.target diff --git a/file_mover/systemctl/demo_mover.service b/file_mover/systemctl/demo_mover.service new file mode 100644 index 00000000..b453628b --- /dev/null +++ b/file_mover/systemctl/demo_mover.service @@ -0,0 +1,8 @@ +[Unit] +Description=Moves demos from the local machine to remote destinations every 5 minutes + +[Service] +Type=simple +User=file_mover +WorkingDirectory=/home/file_mover +ExecStart=/home/file_mover/run_demo_mover.sh diff --git a/file_mover/systemctl/demo_mover.timer b/file_mover/systemctl/demo_mover.timer new file mode 100644 index 00000000..15de12cd --- /dev/null +++ b/file_mover/systemctl/demo_mover.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Moves demos from the local machine to remote destinations every 5 minutes +Requires=demo_mover.service + +[Timer] +OnCalendar=*-*-* *:0,5,10,15,20,25,30,35,40,45,50,55 + +[Install] +WantedBy=multi-user.target