updated further

file_mover/config_backups.json

@@ -1,55 +1,59 @@
 {
 	"remotes":{
-		"local_dir_css_gg":{
+		"local_dir_gameservers": {
-			"description": "local machine gg server",
+			"description": "local machine gameservers",
-                        "path": "/home/gameservers/css_gg/",
+			"path": "/home/gameservers/",
-                        "remote_type": "local_dir"
+			"remote_type": "local_dir"
 		},
-		 "local_dir_css_zr":{
+		"local_dir_var": {
-                        "description": "local machine zr server",
+                        "description": "local machine var",
-                        "path": "/home/gameservers/css_zr/",
+                        "path": "/var/",
                         "remote_type": "local_dir"
                 },
-		 "local_dir_css_mg":{
+		"local_dir_etc": {
-                        "description": "local machine mg server",
+                        "description": "local machine etc",
-                        "path": "/home/gameservers/css_gg/",
+                        "path": "/etc/",
                         "remote_type": "local_dir"
                 },
-		 "local_dir_css_ze":{
+		"sftp_vm_backups_dest_ovh":{
-                        "description": "local machine ze server",
-                        "path": "/home/gameservers/css_ze/",
-                        "remote_type": "local_dir"
-                },
-		 "local_dir_css_jb":{
-                        "description": "local machine jb server",
-                        "path": "/home/gameservers/css_jb/",
-                        "remote_type": "local_dir"
-                },
-		"sftp_vm_backups_dest":{
                         "description": "sftp server for backups, fastdl & demos",
                         "hostname": "163.172.119.53",
                         "username": "nonroot",
                         "port": "22",
-                        "path": "/home/nonroot/backups/",
+                        "path": "/home/nonroot/backups/ovh/",
                         "remote_type": "sftp"
                 }
 	},
 	"jobs":[
 		{
-		"job_name": "backup_local_server_gg",
+		"job_name": "backup_local_gameservers",
-		"job_description": "zips the gg server folder and moves it to backup/fastdl/demo vm",
+		"job_description": "zips the local gameservers folder and moves it to backup/fastdl/demo vm",
-		"src": "local_dir_css_gg",
+		"src": "local_dir_gameservers",
-		"dest": "sftp_vm_backups_dest",
+		"dest": "sftp_vm_backups_dest_ovh",
-		"zipname":"css_gg_backup"
+		"zipname":"ovh_gameservers_backup"
-		}
+		},
+		{
+                "job_name": "backup_local_var",
+                "job_description": "zips the local var folder and moves it to backup/fastdl/demo vm",
+                "src": "local_dir_var",
+                "dest": "sftp_vm_backups_dest_ovh",
+                "zipname":"ovh_var_backup"
+                },
+		{
+                "job_name": "backup_local_etc",
+                "job_description": "zips the local etc folder and moves it to backup/fastdl/demo vm",
+                "src": "local_dir_etc",
+                "dest": "sftp_vm_backups_dest_ovh",
+                "zipname":"ovh_etc_backup"
+                }
 	],
 	"settings": {
 		"sftp": {
 			"remote_attempts": "5",
 			"remote_delay": "15",
-			"163.172.119.53":{
+			"ip address":{
 				"username": {
-					"password": "password123"
+					"password": "password"
 				}
 			}
 		}

file_mover/file_mover.py

@@ -58,9 +58,15 @@ def main():
             distribute_files(source_files, dest)
             for pathfile in source_files:
                 src.delete_local_demo(pathfile)
-            dest.delete_remote(source_files)
+            dest.delete_remote_demos(source_files)
         elif "backup_local" in job["job_name"]:
-            zip_directory_cmd = f'zip -r {job["zipname"]}-{str(datetime.now()).split(" ")[0]}.zip {remotes[job["src"]]["path"]}'
+            zip_file_path = src.zip_local_directory(job)
+            if not dest.put(zip_file_path, dest.path):
+                log_msg = ''.join(["failed putting file: ", str(pathfile)])
+                logging.warning(log_msg)
+                sys.exit(1)
+            src.delete_local_zip(zip_file_path)
+            dest.delete_remote_zips(zip_file_path)
             
         logging.info(('finished job: ', job))
 

file_mover/remote_local_dir.py

@@ -2,24 +2,54 @@ from pathlib import Path
 import contextlib
 import os
 from datetime import timedelta, datetime
+from zipfile import ZipFile
 
 class local_dir_remote:
     def __init__(self, config):
         self.config = config
         self.path = config["path"]
 
-    def list_dir(self):
+    def list_dir(self, recursive_search = False):
         path = Path(self.path)
         file_list = []
         delta = timedelta(minutes=2)
-        for file_demo in list(path.glob('*')):
+        glob_pattern = '*'
-            mtime = datetime.fromtimestamp(os.stat(file_demo).st_mtime)
+        glob_pattern_recursive = '**'
-            if mtime < datetime.now() - delta:
+        if not recursive_search:
-                file_list.append(file_demo)
+            for file_demo in list(path.glob(glob_pattern)):
+                mtime = datetime.fromtimestamp(os.stat(file_demo).st_mtime)
+                if mtime < datetime.now() - delta:
+                    file_list.append(file_demo)
+        else:
+            for file_directory in list(path.glob(glob_pattern_recursive)):
+                try:
+                    for files in list(file_directory.glob(glob_pattern)):
+                        if '/maps' in str(files):
+                            continue
+                        file_list.append(files)
+                except PermissionError:
+                    continue
         return file_list
 
+    def zip_local_directory(self, job):
+        source_files = self.list_dir(True)
+        zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.zip'
+        with ZipFile(zipname, 'w') as myzip:
+            for files in source_files:
+                try:
+                    myzip.write(files)
+                except PermissionError:
+                    continue
+        return str(Path.cwd()) + '/' +  zipname
+
     def delete_local_demo(self, pathfile):
         with contextlib.suppress(FileNotFoundError):
             if str(pathfile).endswith('.dem'):
                 os.remove(pathfile)
                 print('removed demo: ', pathfile)
+
+    def delete_local_zip(self, zip_file_path):
+        with contextlib.suppress(FileNotFoundError):
+            if (str(zip_file_path).endswith('.zip')):
+                os.remove(zip_file_path)
+                print('removed zip: ', zip_file_path)

file_mover/remote_sftp.py

@@ -87,18 +87,31 @@ class sftp_remote:
             if total_attempts == 0:
                 return False
         
+    def change_path(self, pathfile):
+        filename = pathfile.split("/")[-1]
+        pathfile = self.path + filename
+        utime = self.sftp.stat(pathfile).st_mtime
+        last_modified = datetime.fromtimestamp(utime)
+        return pathfile, last_modified
 
-    def delete_remote(self, source_files):
+    def delete_file(self, last_modified, pathfile, delete_time):
+        if (datetime.now() - last_modified) > timedelta(days= delete_time):
+            print('deleting file remotely: ', pathfile)
+            self.sftp.remove(pathfile)
+
+    def delete_remote_demos(self, source_files):
         self.connect()
         for pathfile in source_files:
             pathfile = str(pathfile)
             if not pathfile.endswith('.dem'):
                 continue
-            filename = pathfile.split("/")[-1]
+            pathfile, last_modified = self.change_path(pathfile)
-            pathfile = self.path + filename
+            self.delete_file(last_modified, pathfile, 45)
-            utime = self.sftp.stat(pathfile).st_mtime
+        self.disconnect()
-            last_modified = datetime.fromtimestamp(utime)
+
-            if (datetime.now() - last_modified) > timedelta(days=30 * 1.5):
+    def delete_remote_zips(self, zip_file_path):
-                print('deleting file remotely: ', pathfile)
+        self.connect()
-                self.sftp.remove(pathfile)
+        if zip_file_path.endswith(".zip"):
+            pathfile, last_modified = self.change_path(zip_file_path)
+            self.delete_file(last_modified, pathfile, 10)
         self.disconnect()