UNLOZE/projects-jenz
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

updated backups system

Browse Source
This commit is contained in:
jenz 2023-05-10 22:38:52 +02:00
parent f79327a11f
commit 2385004da4
4 changed files with 173 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

202
file_mover/config_backups.json
View File

@ -15,9 +15,14 @@
"path": "/etc/",
"remote_type": "local_dir"
},
"local_dir_mysqldump":{
"description": "local machine mysqldump",
"path": "",
"remote_type": "local_dir"
},
"sftp_vm_hlstats_src_home":{
"description": "sftp server for hlstats",
"hostname": "51.15.159.31",
"hostname": "5.104.107.39",
"username": "nonroot",
"port": "22",
"path": "/home/",
@ -25,7 +30,7 @@
},
"sftp_vm_hlstats_src_etc":{
"description": "sftp server for hlstats",
"hostname": "51.15.159.31",
"hostname": "5.104.107.39",
"username": "nonroot",
"port": "22",
"path": "/etc/",
@ -33,7 +38,7 @@
},
"sftp_vm_hlstats_src_var":{
"description": "sftp server for hlstats",
"hostname": "51.15.159.31",
"hostname": "5.104.107.39",
"username": "nonroot",
"port": "22",
"path": "/var/",
@ -41,7 +46,7 @@
},
"sftp_vm_hlstats_src_mysqldump":{
"description": "sftp server for hlstats",
"hostname": "51.15.159.31",
"hostname": "5.104.107.39",
"username": "nonroot",
"port": "22",
"path": "",
@ -49,7 +54,7 @@
},
"sftp_vm_xenforo_src_home":{
"description": "sftp server for xenforo",
"hostname": "163.172.225.175",
"hostname": "5.104.107.130",
"username": "nonroot",
"port": "22",
"path": "/home/",
@ -57,7 +62,7 @@
},
"sftp_vm_xenforo_src_etc":{
"description": "sftp server for xenforo",
"hostname": "163.172.225.175",
"hostname": "5.104.107.130",
"username": "nonroot",
"port": "22",
"path": "/etc/",
@ -65,7 +70,7 @@
},
"sftp_vm_xenforo_src_var":{
"description": "sftp server for xenforo",
"hostname": "163.172.225.175",
"hostname": "5.104.107.130",
"username": "nonroot",
"port": "22",
"path": "/var/",
@ -73,7 +78,7 @@
},
"sftp_vm_xenforo_src_mysqldump":{
"description": "sftp server for xenforo",
"hostname": "163.172.225.175",
"hostname": "5.104.107.130",
"username": "nonroot",
"port": "22",
"path": "",
@ -81,7 +86,7 @@
},
"sftp_vm_webservices_src_home":{
"description": "sftp server for webservices",
"hostname": "163.172.117.46",
"hostname": "5.199.136.22",
"username": "nonroot",
"port": "22",
"path": "/home/",
@ -89,7 +94,7 @@
},
"sftp_vm_webservices_src_etc":{
"description": "sftp server for webservices",
"hostname": "163.172.117.46",
"hostname": "5.199.136.22",
"username": "nonroot",
"port": "22",
"path": "/etc/",
@ -97,7 +102,7 @@
},
"sftp_vm_webservices_src_var":{
"description": "sftp server for webservices",
"hostname": "163.172.117.46",
"hostname": "5.199.136.22",
"username": "nonroot",
"port": "22",
"path": "/var/",
@ -105,67 +110,123 @@
},
"sftp_vm_webservices_src_mysqldump":{
"description": "sftp server for webservices",
"hostname": "163.172.117.46",
"hostname": "5.199.136.22",
"username": "nonroot",
"port": "22",
"path": "",
"remote_type": "sftp"
},
"sftp_vm_backups_src_etc":{
"description": "sftp server for backups/fastdl/demos",
"hostname": "163.172.119.53",
"sftp_vm_discord_autism_bot_src_etc":{
"description": "sftp server for autismbot Discord",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/etc/",
"remote_type": "sftp"
},
"sftp_vm_backups_src_var":{
"description": "sftp server for backups/fastdl/demos",
"hostname": "163.172.119.53",
"sftp_vm_discord_autism_bot_src_var":{
"description": "sftp server for autismbot Discord",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/var/",
"remote_type": "sftp"
},
"sftp_vm_discord_autism_bot_src_mysqldump":{
"description": "sftp server for autismbot Discord",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "",
"remote_type": "sftp"
},
"sftp_vm_discord_autism_bot_src_autism_bot":{
"description": "sftp server for autismbot Discord",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/autism_bot/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_ovh":{
"description": "sftp server for backups, fastdl & demos",
"hostname": "163.172.119.53",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/ovh/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_fastdl":{
"description": "sftp server for backups, fastdl & demos",
"hostname": "163.172.119.53",
"sftp_vm_backups2_dest_ovh":{
"description": "Second sftp server for backups, fastdl & demos",
"hostname": "78.31.67.70",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/compressed_backups/ovh/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_discord_autism_bot":{
"description": "sftp server for discord autism bot",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/fastdl/",
"path": "/home/nonroot/backups/autism_bot/",
"remote_type": "sftp"
},
"sftp_vm_backups2_dest_discord_autism_bot":{
"description": "Second sftp server for discord autism bot",
"hostname": "78.31.67.70",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/compressed_backups/autism_bot/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_hlstats":{
"description": "sftp server for backups, fastdl & demos",
"hostname": "163.172.119.53",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/hlstats/",
"remote_type": "sftp"
},
"sftp_vm_backups2_dest_hlstats":{
"description": "Second sftp server for backups, fastdl & demos",
"hostname": "78.31.67.70",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/compressed_backups/hlstats/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_xenforo":{
"description": "sftp server for backups, fastdl & demos",
"hostname": "163.172.119.53",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/xenforo/",
"remote_type": "sftp"
},
"sftp_vm_backups2_dest_xenforo":{
"description": "Second sftp server for backups, fastdl & demos",
"hostname": "78.31.67.70",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/compressed_backups/xenforo/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_webservices":{
"description": "sftp server for backups, fastdl & demos",
"hostname": "163.172.119.53",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/webservices/",
"remote_type": "sftp"
},
"sftp_vm_backups2_dest_webservices":{
"description": "Second sftp server for backups, fastdl & demos",
"hostname": "78.31.67.70",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/compressed_backups/webservices/",
"remote_type": "sftp"
}
},
"jobs":[
@ -174,6 +235,7 @@
"job_description": "zips the remote webservices etc folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_webservices_src_etc",
"dest": "sftp_vm_backups_dest_webservices",
"dest2": "sftp_vm_backups2_dest_webservices",
"zipname":"webservices_etc_backup",
"download_dir": "/home/file_mover/"
},
@ -182,6 +244,7 @@
"job_description": "zips the remote webservices var folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_webservices_src_var",
"dest": "sftp_vm_backups_dest_webservices",
"dest2": "sftp_vm_backups2_dest_webservices",
"zipname":"webservices_var_backup",
"download_dir": "/home/file_mover/"
},
@ -190,6 +253,7 @@
"job_description": "zips the remote webservices home folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_webservices_src_home",
"dest": "sftp_vm_backups_dest_webservices",
"dest2": "sftp_vm_backups2_dest_webservices",
"zipname":"webservices_home_backup",
"download_dir": "/home/file_mover/"
},
@ -198,6 +262,7 @@
"job_description": "dumps the remote webservices vm databases and moves them to backup/fastdl/demo vm",
"src": "sftp_vm_webservices_src_mysqldump",
"dest": "sftp_vm_backups_dest_webservices",
"dest2": "sftp_vm_backups2_dest_webservices",
"zipname":"webservices_mysqldump_backup",
"download_dir": "/home/file_mover/"
},
@ -206,6 +271,7 @@
"job_description": "zips the remote xenforo etc folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_xenforo_src_etc",
"dest": "sftp_vm_backups_dest_xenforo",
"dest2": "sftp_vm_backups2_dest_xenforo",
"zipname":"xenforo_etc_backup",
"download_dir": "/home/file_mover/"
},
@ -214,6 +280,7 @@
"job_description": "zips the remote xenforo var folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_xenforo_src_var",
"dest": "sftp_vm_backups_dest_xenforo",
"dest2": "sftp_vm_backups2_dest_xenforo",
"zipname":"xenforo_var_backup",
"download_dir": "/home/file_mover/"
},
@ -222,6 +289,7 @@
"job_description": "zips the remote xenforo home folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_xenforo_src_home",
"dest": "sftp_vm_backups_dest_xenforo",
"dest2": "sftp_vm_backups2_dest_xenforo",
"zipname":"xenforo_home_backup",
"download_dir": "/home/file_mover/"
},
@ -230,6 +298,7 @@
"job_description": "dumps the remote xenforo vm databases and moves them to backup/fastdl/demo vm",
"src": "sftp_vm_xenforo_src_mysqldump",
"dest": "sftp_vm_backups_dest_xenforo",
"dest2": "sftp_vm_backups2_dest_xenforo",
"zipname":"xenforo_mysqldump_backup",
"download_dir": "/home/file_mover/"
},
@ -238,6 +307,7 @@
"job_description": "zips the remote hlstats etc folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_hlstats_src_etc",
"dest": "sftp_vm_backups_dest_hlstats",
"dest2": "sftp_vm_backups2_dest_hlstats",
"zipname":"hlstats_etc_backup",
"download_dir": "/home/file_mover/"
},
@ -246,6 +316,7 @@
"job_description": "zips the remote hlstats var folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_hlstats_src_var",
"dest": "sftp_vm_backups_dest_hlstats",
"dest2": "sftp_vm_backups2_dest_hlstats",
"zipname":"hlstats_var_backup",
"download_dir": "/home/file_mover/"
},
@ -254,6 +325,7 @@
"job_description": "dumps the remote hlstats vm databases and moves them to backup/fastdl/demo vm",
"src": "sftp_vm_hlstats_src_mysqldump",
"dest": "sftp_vm_backups_dest_hlstats",
"dest2": "sftp_vm_backups2_dest_hlstats",
"zipname":"hlstats_mysqldump_backup",
"download_dir": "/home/file_mover/"
},
@ -262,23 +334,44 @@
"job_description": "zips the remote hlstats home folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_hlstats_src_home",
"dest": "sftp_vm_backups_dest_hlstats",
"dest2": "sftp_vm_backups2_dest_hlstats",
"zipname":"hlstats_home_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_fastdl_etc",
"job_description": "zips the remote backups/fastdl/demos etc folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_backups_src_etc",
"dest": "sftp_vm_backups_dest_fastdl",
"zipname":"fastdl_etc_backup",
"job_name": "backup_remote_autism_bot_etc",
"job_description": "zips the remote discord autism bot etc folder and moves it to discord autism bot vm",
"src": "sftp_vm_discord_autism_bot_src_etc",
"dest": "sftp_vm_backups_dest_discord_autism_bot",
"dest2": "sftp_vm_backups2_dest_discord_autism_bot",
"zipname":"discord_autism_bot_etc_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_fastdl_var",
"job_description": "zips the remote backups/fastdl/demos var folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_backups_src_var",
"dest": "sftp_vm_backups_dest_fastdl",
"zipname":"fastdl_var_backup",
{
"job_name": "backup_remote_autism_bot_var",
"job_description": "zips the remote discord autism bot var folder and moves it to discord autism bot vm",
"src": "sftp_vm_discord_autism_bot_src_var",
"dest": "sftp_vm_backups_dest_discord_autism_bot",
"dest2": "sftp_vm_backups2_dest_discord_autism_bot",
"zipname":"discord_autism_bot_var_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_autism_bot_mysqldump",
"job_description": "dumps the remote discord autism bot vm databases and moves them to discord autism bot vm",
"src": "sftp_vm_discord_autism_bot_src_mysqldump",
"dest": "sftp_vm_backups_dest_discord_autism_bot",
"dest2": "sftp_vm_backups2_dest_discord_autism_bot",
"zipname":"discord_autism_bot_mysqldump_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_autism_bot_discord",
"job_description": "zips the remote autism bot discord autism bot discord folder and moves it to backup/demo vm",
"src": "sftp_vm_discord_autism_bot_src_autism_bot",
"dest": "sftp_vm_backups_dest_discord_autism_bot",
"dest2": "sftp_vm_backups2_dest_discord_autism_bot",
"zipname":"discord_autism_bot_folder_backup",
"download_dir": "/home/file_mover/"
},
{
@ -286,6 +379,7 @@
"job_description": "zips the local gameservers folder and moves it to backup/fastdl/demo vm",
"src": "local_dir_gameservers",
"dest": "sftp_vm_backups_dest_ovh",
"dest2": "sftp_vm_backups2_dest_ovh",
"zipname":"ovh_gameservers_backup"
},
{
@ -293,6 +387,7 @@
"job_description": "zips the local var folder and moves it to backup/fastdl/demo vm",
"src": "local_dir_var",
"dest": "sftp_vm_backups_dest_ovh",
"dest2": "sftp_vm_backups2_dest_ovh",
"zipname":"ovh_var_backup"
},
{
@ -300,33 +395,48 @@
"job_description": "zips the local etc folder and moves it to backup/fastdl/demo vm",
"src": "local_dir_etc",
"dest": "sftp_vm_backups_dest_ovh",
"dest2": "sftp_vm_backups2_dest_ovh",
"zipname":"ovh_etc_backup"
},
{
"job_name": "backup_local_mysqldump",
"job_description": "zips the local databases and moves it to backup/fastdl/demo vm",
"src": "local_dir_mysqldump",
"dest": "sftp_vm_backups_dest_ovh",
"dest2": "sftp_vm_backups2_dest_ovh",
"zipname":"ovh_mysqldump_backup"
}
],
"settings": {
"sftp": {
"remote_attempts": "5",
"remote_delay": "15",
"ip":{
"5.199.136.34":{
"username": {
"password": "password"
"password": ""
}
},
"ip":{
"5.199.136.22":{
"username": {
"password": "password"
"password": ""
}
},
"ip":{
"5.104.107.130":{
"username": {
"password": "password"
"password": ""
}
},
"ip":{
"5.104.107.39":{
"username": {
"password": "password"
"password": ""
}
}
},
"78.31.67.70": {
"username": {
"password": ""
}
}
}
}
}

14
file_mover/file_mover.py
View File

@ -53,9 +53,11 @@ def main():
for index, job in enumerate(jobs):
src = create_remote(remotes[job["src"]], settings)
dest = create_remote(remotes[job["dest"]], settings)
dest2 = create_remote(remotes[job["dest2"]], settings)
if index == 0:
#delete old files (not very adaptable, cant handle dirs in dirs)
dest.delete_remote_zips("/home/nonroot/backups")
dest2.delete_remote_zips("/home/autismbot5/compressed_backups")
if 'demo_' in job["job_name"]:
source_files = src.list_dir()
distribute_files(source_files, dest)
@ -68,6 +70,10 @@ def main():
log_msg = ''.join(["failed putting file: ", str(pathfile)])
logging.warning(log_msg)
sys.exit(1)
if not dest2.put(zip_file_path, dest2.path):
log_msg = ''.join(["failed putting file: ", str(pathfile)])
logging.warning(log_msg)
sys.exit(1)
src.delete_local_zip(zip_file_path)
elif "mysqldump" in job["job_name"]:
src.remote_channel_command(job, "mysqldump")
@ -85,6 +91,10 @@ def main():
logging.warning(f'failed putting local zip {local_zip_path_name} at job {job}')
src.delete_local_zip(local_zip_path_name)
sys.exit(1)
if not dest2.put(local_zip_path_name, dest2.path):
logging.warning(f'failed putting local zip {local_zip_path_name} at job {job}')
src.delete_local_zip(local_zip_path_name)
sys.exit(1)
src.delete_local_zip(local_zip_path_name)
elif "backup_remote" in job["job_name"]:
zipname = src.remote_channel_command(job, "tar_recursive")
@ -101,6 +111,10 @@ def main():
logging.warning(f'failed putting local zip {local_zip_path_name} at job {job}')
src.delete_local_zip(local_zip_path_name)
sys.exit(1)
if not dest2.put(local_zip_path_name, dest2.path):
logging.warning(f'failed putting local zip {local_zip_path_name} at job {job}')
src.delete_local_zip(local_zip_path_name)
sys.exit(1)
src.delete_local_zip(local_zip_path_name)
logging.info(('finished job: ', job))

1
file_mover/remote_local_dir.py
View File

@ -53,3 +53,4 @@ class local_dir_remote:
if (str(zip_file_path).endswith('.tar.xz')):
os.remove(zip_file_path)
print('removed tar xz: ', zip_file_path)

3
file_mover/remote_sftp.py
View File

@ -133,7 +133,7 @@ class sftp_remote:
local_temp_folder = local_path_get + "/tempfolder/"
try:
self.connect()
#print('remote_path chdir: ', remote_path)
print('remote_path chdir: ', remote_path)
self.sftp.chdir(remote_path) # Test if remote_path exists
except IOError:
self.sftp.mkdir(remote_path)
@ -204,3 +204,4 @@ class sftp_remote:
def delete_local_zip(self, local_zip_path_name):
os.remove(local_zip_path_name)