slight changes to adapt to new place

This commit is contained in:
jenz 2023-06-14 23:45:48 +02:00
parent 73be3c4904
commit 8542e72a2c
4 changed files with 76 additions and 441 deletions

View File

@ -5,434 +5,78 @@
"path": "/home/gameservers/", "path": "/home/gameservers/",
"remote_type": "local_dir" "remote_type": "local_dir"
}, },
"local_dir_var": { "sftp_hetzner_src_home":{
"description": "local machine var", "description": "sftp server hetzner",
"path": "/var/", "hostname": "iphere",
"remote_type": "local_dir" "username": "nonroot",
}, "port": "22",
"local_dir_etc": { "path": "/home/",
"description": "local machine etc", "remote_type": "sftp"
"path": "/etc/", },
"remote_type": "local_dir" "sftp_hetzner_src_mysqldump":{
}, "description": "sftp server hetzner",
"local_dir_mysqldump":{ "hostname": "iphere",
"description": "local machine mysqldump", "username": "nonroot",
"path": "", "port": "22",
"remote_type": "local_dir" "path": "",
}, "remote_type": "sftp"
"sftp_vm_hlstats_src_home":{ },
"description": "sftp server for hlstats", "sftp_dest_hetzner":{
"hostname": "5.104.107.39", "description": "sftp server hetzner",
"username": "nonroot", "hostname": "iphere",
"port": "22", "username": "nonroot",
"path": "/home/", "port": "22",
"remote_type": "sftp" "path": "/home/nonroot/backups/",
}, "remote_type": "sftp"
"sftp_vm_hlstats_src_etc":{ },
"description": "sftp server for hlstats", "sftp_dest_backups2":{
"hostname": "5.104.107.39", "description": "Second sftp server",
"username": "nonroot", "hostname": "iphere",
"port": "22", "username": "autismbot5",
"path": "/etc/", "port": "22",
"remote_type": "sftp" "path": "/home/autismbot5/backups/",
}, "remote_type": "sftp"
"sftp_vm_hlstats_src_var":{ }
"description": "sftp server for hlstats",
"hostname": "5.104.107.39",
"username": "nonroot",
"port": "22",
"path": "/var/",
"remote_type": "sftp"
},
"sftp_vm_hlstats_src_mysqldump":{
"description": "sftp server for hlstats",
"hostname": "5.104.107.39",
"username": "nonroot",
"port": "22",
"path": "",
"remote_type": "sftp"
},
"sftp_vm_xenforo_src_home":{
"description": "sftp server for xenforo",
"hostname": "5.104.107.130",
"username": "nonroot",
"port": "22",
"path": "/home/",
"remote_type": "sftp"
},
"sftp_vm_xenforo_src_etc":{
"description": "sftp server for xenforo",
"hostname": "5.104.107.130",
"username": "nonroot",
"port": "22",
"path": "/etc/",
"remote_type": "sftp"
},
"sftp_vm_xenforo_src_var":{
"description": "sftp server for xenforo",
"hostname": "5.104.107.130",
"username": "nonroot",
"port": "22",
"path": "/var/",
"remote_type": "sftp"
},
"sftp_vm_xenforo_src_mysqldump":{
"description": "sftp server for xenforo",
"hostname": "5.104.107.130",
"username": "nonroot",
"port": "22",
"path": "",
"remote_type": "sftp"
},
"sftp_vm_webservices_src_home":{
"description": "sftp server for webservices",
"hostname": "5.199.136.22",
"username": "nonroot",
"port": "22",
"path": "/home/",
"remote_type": "sftp"
},
"sftp_vm_webservices_src_etc":{
"description": "sftp server for webservices",
"hostname": "5.199.136.22",
"username": "nonroot",
"port": "22",
"path": "/etc/",
"remote_type": "sftp"
},
"sftp_vm_webservices_src_var":{
"description": "sftp server for webservices",
"hostname": "5.199.136.22",
"username": "nonroot",
"port": "22",
"path": "/var/",
"remote_type": "sftp"
},
"sftp_vm_webservices_src_mysqldump":{
"description": "sftp server for webservices",
"hostname": "5.199.136.22",
"username": "nonroot",
"port": "22",
"path": "",
"remote_type": "sftp"
},
"sftp_vm_discord_autism_bot_src_etc":{
"description": "sftp server for autismbot Discord",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/etc/",
"remote_type": "sftp"
},
"sftp_vm_discord_autism_bot_src_var":{
"description": "sftp server for autismbot Discord",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/var/",
"remote_type": "sftp"
},
"sftp_vm_discord_autism_bot_src_mysqldump":{
"description": "sftp server for autismbot Discord",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "",
"remote_type": "sftp"
},
"sftp_vm_discord_autism_bot_src_autism_bot":{
"description": "sftp server for autismbot Discord",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/autism_bot/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_ovh":{
"description": "sftp server for backups, fastdl & demos",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/ovh/",
"remote_type": "sftp"
},
"sftp_vm_backups2_dest_ovh":{
"description": "Second sftp server for backups, fastdl & demos",
"hostname": "78.31.67.70",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/compressed_backups/ovh/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_discord_autism_bot":{
"description": "sftp server for discord autism bot",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/autism_bot/",
"remote_type": "sftp"
},
"sftp_vm_backups2_dest_discord_autism_bot":{
"description": "Second sftp server for discord autism bot",
"hostname": "78.31.67.70",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/compressed_backups/autism_bot/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_hlstats":{
"description": "sftp server for backups, fastdl & demos",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/hlstats/",
"remote_type": "sftp"
},
"sftp_vm_backups2_dest_hlstats":{
"description": "Second sftp server for backups, fastdl & demos",
"hostname": "78.31.67.70",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/compressed_backups/hlstats/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_xenforo":{
"description": "sftp server for backups, fastdl & demos",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/xenforo/",
"remote_type": "sftp"
},
"sftp_vm_backups2_dest_xenforo":{
"description": "Second sftp server for backups, fastdl & demos",
"hostname": "78.31.67.70",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/compressed_backups/xenforo/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_webservices":{
"description": "sftp server for backups, fastdl & demos",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/webservices/",
"remote_type": "sftp"
},
"sftp_vm_backups2_dest_webservices":{
"description": "Second sftp server for backups, fastdl & demos",
"hostname": "78.31.67.70",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/compressed_backups/webservices/",
"remote_type": "sftp"
}
}, },
"jobs":[ "jobs":[
{ {
"job_name": "backup_remote_webservices_etc", "job_name": "backup_remote_hetzner_home",
"job_description": "zips the remote webservices etc folder and moves it to backup/fastdl/demo vm", "job_description": "zips hetzner home directory",
"src": "sftp_vm_webservices_src_etc", "src": "sftp_hetzner_src_home",
"dest": "sftp_vm_backups_dest_webservices", "dest": "sftp_dest_hetzner",
"dest2": "sftp_vm_backups2_dest_webservices", "dest2": "sftp_dest_backups2",
"zipname":"webservices_etc_backup", "zipname":"hetzner_home_dir",
"download_dir": "/home/file_mover/" "download_dir": "/home/file_mover/"
}, },
{ {
"job_name": "backup_remote_webservices_var", "job_name": "hetzner_mysqldump",
"job_description": "zips the remote webservices var folder and moves it to backup/fastdl/demo vm", "job_description": "dumps hetzner databases",
"src": "sftp_vm_webservices_src_var", "src": "sftp_hetzner_src_mysqldump",
"dest": "sftp_vm_backups_dest_webservices", "dest": "sftp_dest_hetzner",
"dest2": "sftp_vm_backups2_dest_webservices", "dest2": "sftp_dest_backups2",
"zipname":"webservices_var_backup", "zipname":"hetzner_mysqldump",
"download_dir": "/home/file_mover/" "download_dir": "/home/file_mover/"
}, },
{
"job_name": "backup_remote_webservices_home",
"job_description": "zips the remote webservices home folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_webservices_src_home",
"dest": "sftp_vm_backups_dest_webservices",
"dest2": "sftp_vm_backups2_dest_webservices",
"zipname":"webservices_home_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_webservices_mysqldump",
"job_description": "dumps the remote webservices vm databases and moves them to backup/fastdl/demo vm",
"src": "sftp_vm_webservices_src_mysqldump",
"dest": "sftp_vm_backups_dest_webservices",
"dest2": "sftp_vm_backups2_dest_webservices",
"zipname":"webservices_mysqldump_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_xenforo_etc",
"job_description": "zips the remote xenforo etc folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_xenforo_src_etc",
"dest": "sftp_vm_backups_dest_xenforo",
"dest2": "sftp_vm_backups2_dest_xenforo",
"zipname":"xenforo_etc_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_xenforo_var",
"job_description": "zips the remote xenforo var folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_xenforo_src_var",
"dest": "sftp_vm_backups_dest_xenforo",
"dest2": "sftp_vm_backups2_dest_xenforo",
"zipname":"xenforo_var_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_xenforo_home",
"job_description": "zips the remote xenforo home folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_xenforo_src_home",
"dest": "sftp_vm_backups_dest_xenforo",
"dest2": "sftp_vm_backups2_dest_xenforo",
"zipname":"xenforo_home_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_xenforo_mysqldump",
"job_description": "dumps the remote xenforo vm databases and moves them to backup/fastdl/demo vm",
"src": "sftp_vm_xenforo_src_mysqldump",
"dest": "sftp_vm_backups_dest_xenforo",
"dest2": "sftp_vm_backups2_dest_xenforo",
"zipname":"xenforo_mysqldump_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_hlstats_etc",
"job_description": "zips the remote hlstats etc folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_hlstats_src_etc",
"dest": "sftp_vm_backups_dest_hlstats",
"dest2": "sftp_vm_backups2_dest_hlstats",
"zipname":"hlstats_etc_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_hlstats_var",
"job_description": "zips the remote hlstats var folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_hlstats_src_var",
"dest": "sftp_vm_backups_dest_hlstats",
"dest2": "sftp_vm_backups2_dest_hlstats",
"zipname":"hlstats_var_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_hlstats_mysqldump",
"job_description": "dumps the remote hlstats vm databases and moves them to backup/fastdl/demo vm",
"src": "sftp_vm_hlstats_src_mysqldump",
"dest": "sftp_vm_backups_dest_hlstats",
"dest2": "sftp_vm_backups2_dest_hlstats",
"zipname":"hlstats_mysqldump_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_hlstats_home",
"job_description": "zips the remote hlstats home folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_hlstats_src_home",
"dest": "sftp_vm_backups_dest_hlstats",
"dest2": "sftp_vm_backups2_dest_hlstats",
"zipname":"hlstats_home_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_autism_bot_etc",
"job_description": "zips the remote discord autism bot etc folder and moves it to discord autism bot vm",
"src": "sftp_vm_discord_autism_bot_src_etc",
"dest": "sftp_vm_backups_dest_discord_autism_bot",
"dest2": "sftp_vm_backups2_dest_discord_autism_bot",
"zipname":"discord_autism_bot_etc_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_autism_bot_var",
"job_description": "zips the remote discord autism bot var folder and moves it to discord autism bot vm",
"src": "sftp_vm_discord_autism_bot_src_var",
"dest": "sftp_vm_backups_dest_discord_autism_bot",
"dest2": "sftp_vm_backups2_dest_discord_autism_bot",
"zipname":"discord_autism_bot_var_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_autism_bot_mysqldump",
"job_description": "dumps the remote discord autism bot vm databases and moves them to discord autism bot vm",
"src": "sftp_vm_discord_autism_bot_src_mysqldump",
"dest": "sftp_vm_backups_dest_discord_autism_bot",
"dest2": "sftp_vm_backups2_dest_discord_autism_bot",
"zipname":"discord_autism_bot_mysqldump_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_autism_bot_discord",
"job_description": "zips the remote autism bot discord autism bot discord folder and moves it to backup/demo vm",
"src": "sftp_vm_discord_autism_bot_src_autism_bot",
"dest": "sftp_vm_backups_dest_discord_autism_bot",
"dest2": "sftp_vm_backups2_dest_discord_autism_bot",
"zipname":"discord_autism_bot_folder_backup",
"download_dir": "/home/file_mover/"
},
{ {
"job_name": "backup_local_gameservers", "job_name": "backup_local_gameservers",
"job_description": "zips the local gameservers folder and moves it to backup/fastdl/demo vm", "job_description": "zips the local gameservers folder",
"src": "local_dir_gameservers", "src": "local_dir_gameservers",
"dest": "sftp_vm_backups_dest_ovh", "dest": "sftp_dest_hetzner",
"dest2": "sftp_vm_backups2_dest_ovh", "dest2": "sftp_dest_backups2",
"zipname":"ovh_gameservers_backup" "zipname":"ovh_gameservers"
}, }
{
"job_name": "backup_local_var",
"job_description": "zips the local var folder and moves it to backup/fastdl/demo vm",
"src": "local_dir_var",
"dest": "sftp_vm_backups_dest_ovh",
"dest2": "sftp_vm_backups2_dest_ovh",
"zipname":"ovh_var_backup"
},
{
"job_name": "backup_local_etc",
"job_description": "zips the local etc folder and moves it to backup/fastdl/demo vm",
"src": "local_dir_etc",
"dest": "sftp_vm_backups_dest_ovh",
"dest2": "sftp_vm_backups2_dest_ovh",
"zipname":"ovh_etc_backup"
},
{
"job_name": "backup_local_mysqldump",
"job_description": "zips the local databases and moves it to backup/fastdl/demo vm",
"src": "local_dir_mysqldump",
"dest": "sftp_vm_backups_dest_ovh",
"dest2": "sftp_vm_backups2_dest_ovh",
"zipname":"ovh_mysqldump_backup"
}
], ],
"settings": { "settings": {
"sftp": { "sftp": {
"remote_attempts": "5", "remote_attempts": "5",
"remote_delay": "15", "remote_delay": "15",
"5.199.136.34":{ "iphere":{
"username": { "nonroot": {
"password": "" "password": ""
} }
}, },
"5.199.136.22":{ "iphere": {
"username": { "autismbot5": {
"password": ""
}
},
"5.104.107.130":{
"username": {
"password": ""
}
},
"5.104.107.39":{
"username": {
"password": ""
}
},
"78.31.67.70": {
"username": {
"password": "" "password": ""
} }
} }

View File

@ -45,8 +45,8 @@ def load_config(config_file):
sys.exit(1) sys.exit(1)
def main(): def main():
#runs demos every 5 minutes but backups thrice a week with systemctl services/timers #runs backups
config_file = 'config_demos.json' config_file = 'config_backups.json'
if sys.argv[1:]: if sys.argv[1:]:
config_file = sys.argv[1] config_file = sys.argv[1]
remotes, jobs, settings = load_config(config_file) remotes, jobs, settings = load_config(config_file)
@ -56,15 +56,9 @@ def main():
dest2 = create_remote(remotes[job["dest2"]], settings) dest2 = create_remote(remotes[job["dest2"]], settings)
if index == 0: if index == 0:
#delete old files (not very adaptable, cant handle dirs in dirs) #delete old files (not very adaptable, cant handle dirs in dirs)
dest.delete_remote_zips("/home/nonroot/backups") dest.delete_remote_zips("/home/nonroot/backups/")
dest2.delete_remote_zips("/home/autismbot5/compressed_backups") dest2.delete_remote_zips("/home/autismbot5/backups/")
if 'demo_' in job["job_name"]: if "backup_local" in job["job_name"]:
source_files = src.list_dir()
distribute_files(source_files, dest)
for pathfile in source_files:
src.delete_local_demo(pathfile)
dest.delete_remote_demos(source_files)
elif "backup_local" in job["job_name"]:
zip_file_path = src.zip_local_directory(job) zip_file_path = src.zip_local_directory(job)
if not dest.put(zip_file_path, dest.path): if not dest.put(zip_file_path, dest.path):
log_msg = ''.join(["failed putting file: ", str(pathfile)]) log_msg = ''.join(["failed putting file: ", str(pathfile)])
@ -120,3 +114,4 @@ def main():
if __name__ == '__main__': if __name__ == '__main__':
main() main()

View File

@ -1,7 +1,3 @@
from pathlib import Path
import contextlib
import os
from datetime import timedelta, datetime
import subprocess import subprocess
class local_dir_remote: class local_dir_remote:

View File

@ -50,11 +50,12 @@ class sftp_remote:
channel = self.ssh.get_transport().open_session(timeout=120) channel = self.ssh.get_transport().open_session(timeout=120)
#-T3= 3 threads, use 70% of available ram #-T3= 3 threads, use 70% of available ram
#TODO exclude dir where hlstats spams pictures of the player graphs
if 'tar_recursive' == command_state: if 'tar_recursive' == command_state:
exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar --exclude={zipname} -cJf {zipname} {self.path}" exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar --exclude={zipname} --exclude=/home/nonroot/backups/ --exclude=/home/nonroot/demos --exclude=/home/nonroot/snap --exclude=logs/ --ignore-failed-read -cJf {zipname} {self.path}"
elif 'mysqldump' == command_state: elif 'mysqldump' == command_state:
#check README for my.cnf regarding how this works #check README for my.cnf regarding how this works
exec_command_state = f'mysqldump -u root --all-databases > mysqldump.sql' exec_command_state = f'mysqldump -u backups_db --all-databases > mysqldump.sql'
else: #tar_mysqldump else: #tar_mysqldump
exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar -cJf {zipname} /home/{self.username}/mysqldump.sql" exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar -cJf {zipname} /home/{self.username}/mysqldump.sql"
exec_cmd = f'cd /home/{self.username}/; {exec_command_state}' exec_cmd = f'cd /home/{self.username}/; {exec_command_state}'
@ -187,14 +188,13 @@ class sftp_remote:
def delete_remote_zips(self, backups_dir): def delete_remote_zips(self, backups_dir):
self.connect() self.connect()
for directories in self.sftp.listdir(backups_dir): for files in self.sftp.listdir(f'{backups_dir}'):
for files in self.sftp.listdir(f'{backups_dir}/{directories}'): pathfile = f'{backups_dir}{files}'
pathfile = f'{backups_dir}/{directories}/{files}' if not pathfile.endswith(".xz"):
if not pathfile.endswith(".xz"): continue
continue utime = self.sftp.stat(pathfile).st_mtime
utime = self.sftp.stat(pathfile).st_mtime last_modified = datetime.fromtimestamp(utime)
last_modified = datetime.fromtimestamp(utime) self.delete_file(last_modified, pathfile, 40)
self.delete_file(last_modified, pathfile, 25)
self.disconnect() self.disconnect()
def delete_remote_zip_temp(self, zipname): def delete_remote_zip_temp(self, zipname):