From 10d0323add811204228f73072705bda053827afe Mon Sep 17 00:00:00 2001 From: christian Date: Sat, 30 Oct 2021 23:52:37 +0200 Subject: [PATCH] updated further --- file_mover/config_backups.json | 64 +++++++++++++------------ file_mover/config_demos.json | 4 +- file_mover/file_mover.py | 12 +++-- file_mover/remote_local_dir.py | 40 ++++++++++++++-- file_mover/remote_sftp.py | 29 +++++++---- file_mover/systemctl/backups.service | 0 file_mover/systemctl/backups.timer | 0 file_mover/systemctl/demo_mover.service | 0 file_mover/systemctl/demo_mover.timer | 0 9 files changed, 101 insertions(+), 48 deletions(-) mode change 100644 => 100755 file_mover/systemctl/backups.service mode change 100644 => 100755 file_mover/systemctl/backups.timer mode change 100644 => 100755 file_mover/systemctl/demo_mover.service mode change 100644 => 100755 file_mover/systemctl/demo_mover.timer diff --git a/file_mover/config_backups.json b/file_mover/config_backups.json index f8de0eb2..3a1450d8 100755 --- a/file_mover/config_backups.json +++ b/file_mover/config_backups.json @@ -1,55 +1,59 @@ { "remotes":{ - "local_dir_css_gg":{ - "description": "local machine gg server", - "path": "/home/gameservers/css_gg/", - "remote_type": "local_dir" + "local_dir_gameservers": { + "description": "local machine gameservers", + "path": "/home/gameservers/", + "remote_type": "local_dir" }, - "local_dir_css_zr":{ - "description": "local machine zr server", - "path": "/home/gameservers/css_zr/", + "local_dir_var": { + "description": "local machine var", + "path": "/var/", "remote_type": "local_dir" }, - "local_dir_css_mg":{ - "description": "local machine mg server", - "path": "/home/gameservers/css_gg/", + "local_dir_etc": { + "description": "local machine etc", + "path": "/etc/", "remote_type": "local_dir" }, - "local_dir_css_ze":{ - "description": "local machine ze server", - "path": "/home/gameservers/css_ze/", - "remote_type": "local_dir" - }, - "local_dir_css_jb":{ - "description": "local machine jb server", - "path": "/home/gameservers/css_jb/", - "remote_type": "local_dir" - }, - "sftp_vm_backups_dest":{ + "sftp_vm_backups_dest_ovh":{ "description": "sftp server for backups, fastdl & demos", "hostname": "163.172.119.53", "username": "nonroot", "port": "22", - "path": "/home/nonroot/backups/", + "path": "/home/nonroot/backups/ovh/", "remote_type": "sftp" } }, "jobs":[ { - "job_name": "backup_local_server_gg", - "job_description": "zips the gg server folder and moves it to backup/fastdl/demo vm", - "src": "local_dir_css_gg", - "dest": "sftp_vm_backups_dest", - "zipname":"css_gg_backup" - } + "job_name": "backup_local_gameservers", + "job_description": "zips the local gameservers folder and moves it to backup/fastdl/demo vm", + "src": "local_dir_gameservers", + "dest": "sftp_vm_backups_dest_ovh", + "zipname":"ovh_gameservers_backup" + }, + { + "job_name": "backup_local_var", + "job_description": "zips the local var folder and moves it to backup/fastdl/demo vm", + "src": "local_dir_var", + "dest": "sftp_vm_backups_dest_ovh", + "zipname":"ovh_var_backup" + }, + { + "job_name": "backup_local_etc", + "job_description": "zips the local etc folder and moves it to backup/fastdl/demo vm", + "src": "local_dir_etc", + "dest": "sftp_vm_backups_dest_ovh", + "zipname":"ovh_etc_backup" + } ], "settings": { "sftp": { "remote_attempts": "5", "remote_delay": "15", - "163.172.119.53":{ + "ip address":{ "username": { - "password": "password123" + "password": "password" } } } diff --git a/file_mover/config_demos.json b/file_mover/config_demos.json index ee7427e8..d2a6a685 100755 --- a/file_mover/config_demos.json +++ b/file_mover/config_demos.json @@ -37,9 +37,9 @@ "sftp": { "remote_attempts": "5", "remote_delay": "15", - "163.172.119.53":{ + "ip address":{ "username": { - "password": "password123" + "password": "password" } } } diff --git a/file_mover/file_mover.py b/file_mover/file_mover.py index 10481d32..c657e92f 100755 --- a/file_mover/file_mover.py +++ b/file_mover/file_mover.py @@ -58,10 +58,16 @@ def main(): distribute_files(source_files, dest) for pathfile in source_files: src.delete_local_demo(pathfile) - dest.delete_remote(source_files) + dest.delete_remote_demos(source_files) elif "backup_local" in job["job_name"]: - zip_directory_cmd = f'zip -r {job["zipname"]}-{str(datetime.now()).split(" ")[0]}.zip {remotes[job["src"]]["path"]}' - + zip_file_path = src.zip_local_directory(job) + if not dest.put(zip_file_path, dest.path): + log_msg = ''.join(["failed putting file: ", str(pathfile)]) + logging.warning(log_msg) + sys.exit(1) + src.delete_local_zip(zip_file_path) + dest.delete_remote_zips(zip_file_path) + logging.info(('finished job: ', job)) if __name__ == '__main__': diff --git a/file_mover/remote_local_dir.py b/file_mover/remote_local_dir.py index de132911..f25cb4b9 100755 --- a/file_mover/remote_local_dir.py +++ b/file_mover/remote_local_dir.py @@ -2,24 +2,54 @@ from pathlib import Path import contextlib import os from datetime import timedelta, datetime +from zipfile import ZipFile class local_dir_remote: def __init__(self, config): self.config = config self.path = config["path"] - def list_dir(self): + def list_dir(self, recursive_search = False): path = Path(self.path) file_list = [] delta = timedelta(minutes=2) - for file_demo in list(path.glob('*')): - mtime = datetime.fromtimestamp(os.stat(file_demo).st_mtime) - if mtime < datetime.now() - delta: - file_list.append(file_demo) + glob_pattern = '*' + glob_pattern_recursive = '**' + if not recursive_search: + for file_demo in list(path.glob(glob_pattern)): + mtime = datetime.fromtimestamp(os.stat(file_demo).st_mtime) + if mtime < datetime.now() - delta: + file_list.append(file_demo) + else: + for file_directory in list(path.glob(glob_pattern_recursive)): + try: + for files in list(file_directory.glob(glob_pattern)): + if '/maps' in str(files): + continue + file_list.append(files) + except PermissionError: + continue return file_list + def zip_local_directory(self, job): + source_files = self.list_dir(True) + zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.zip' + with ZipFile(zipname, 'w') as myzip: + for files in source_files: + try: + myzip.write(files) + except PermissionError: + continue + return str(Path.cwd()) + '/' + zipname + def delete_local_demo(self, pathfile): with contextlib.suppress(FileNotFoundError): if str(pathfile).endswith('.dem'): os.remove(pathfile) print('removed demo: ', pathfile) + + def delete_local_zip(self, zip_file_path): + with contextlib.suppress(FileNotFoundError): + if (str(zip_file_path).endswith('.zip')): + os.remove(zip_file_path) + print('removed zip: ', zip_file_path) diff --git a/file_mover/remote_sftp.py b/file_mover/remote_sftp.py index a097fded..090aa132 100755 --- a/file_mover/remote_sftp.py +++ b/file_mover/remote_sftp.py @@ -87,18 +87,31 @@ class sftp_remote: if total_attempts == 0: return False + def change_path(self, pathfile): + filename = pathfile.split("/")[-1] + pathfile = self.path + filename + utime = self.sftp.stat(pathfile).st_mtime + last_modified = datetime.fromtimestamp(utime) + return pathfile, last_modified - def delete_remote(self, source_files): + def delete_file(self, last_modified, pathfile, delete_time): + if (datetime.now() - last_modified) > timedelta(days= delete_time): + print('deleting file remotely: ', pathfile) + self.sftp.remove(pathfile) + + def delete_remote_demos(self, source_files): self.connect() for pathfile in source_files: pathfile = str(pathfile) if not pathfile.endswith('.dem'): continue - filename = pathfile.split("/")[-1] - pathfile = self.path + filename - utime = self.sftp.stat(pathfile).st_mtime - last_modified = datetime.fromtimestamp(utime) - if (datetime.now() - last_modified) > timedelta(days=30 * 1.5): - print('deleting file remotely: ', pathfile) - self.sftp.remove(pathfile) + pathfile, last_modified = self.change_path(pathfile) + self.delete_file(last_modified, pathfile, 45) + self.disconnect() + + def delete_remote_zips(self, zip_file_path): + self.connect() + if zip_file_path.endswith(".zip"): + pathfile, last_modified = self.change_path(zip_file_path) + self.delete_file(last_modified, pathfile, 10) self.disconnect() diff --git a/file_mover/systemctl/backups.service b/file_mover/systemctl/backups.service old mode 100644 new mode 100755 diff --git a/file_mover/systemctl/backups.timer b/file_mover/systemctl/backups.timer old mode 100644 new mode 100755 diff --git a/file_mover/systemctl/demo_mover.service b/file_mover/systemctl/demo_mover.service old mode 100644 new mode 100755 diff --git a/file_mover/systemctl/demo_mover.timer b/file_mover/systemctl/demo_mover.timer old mode 100644 new mode 100755