slight changes to adapt to new place

This commit is contained in:
jenz 2023-06-14 23:45:48 +02:00
parent 73be3c4904
commit 8542e72a2c
4 changed files with 76 additions and 441 deletions

View File

@ -5,434 +5,78 @@
"path": "/home/gameservers/",
"remote_type": "local_dir"
},
"local_dir_var": {
"description": "local machine var",
"path": "/var/",
"remote_type": "local_dir"
},
"local_dir_etc": {
"description": "local machine etc",
"path": "/etc/",
"remote_type": "local_dir"
},
"local_dir_mysqldump":{
"description": "local machine mysqldump",
"path": "",
"remote_type": "local_dir"
},
"sftp_vm_hlstats_src_home":{
"description": "sftp server for hlstats",
"hostname": "5.104.107.39",
"username": "nonroot",
"port": "22",
"path": "/home/",
"remote_type": "sftp"
},
"sftp_vm_hlstats_src_etc":{
"description": "sftp server for hlstats",
"hostname": "5.104.107.39",
"username": "nonroot",
"port": "22",
"path": "/etc/",
"remote_type": "sftp"
},
"sftp_vm_hlstats_src_var":{
"description": "sftp server for hlstats",
"hostname": "5.104.107.39",
"username": "nonroot",
"port": "22",
"path": "/var/",
"remote_type": "sftp"
},
"sftp_vm_hlstats_src_mysqldump":{
"description": "sftp server for hlstats",
"hostname": "5.104.107.39",
"username": "nonroot",
"port": "22",
"path": "",
"remote_type": "sftp"
},
"sftp_vm_xenforo_src_home":{
"description": "sftp server for xenforo",
"hostname": "5.104.107.130",
"username": "nonroot",
"port": "22",
"path": "/home/",
"remote_type": "sftp"
},
"sftp_vm_xenforo_src_etc":{
"description": "sftp server for xenforo",
"hostname": "5.104.107.130",
"username": "nonroot",
"port": "22",
"path": "/etc/",
"remote_type": "sftp"
},
"sftp_vm_xenforo_src_var":{
"description": "sftp server for xenforo",
"hostname": "5.104.107.130",
"username": "nonroot",
"port": "22",
"path": "/var/",
"remote_type": "sftp"
},
"sftp_vm_xenforo_src_mysqldump":{
"description": "sftp server for xenforo",
"hostname": "5.104.107.130",
"username": "nonroot",
"port": "22",
"path": "",
"remote_type": "sftp"
},
"sftp_vm_webservices_src_home":{
"description": "sftp server for webservices",
"hostname": "5.199.136.22",
"username": "nonroot",
"port": "22",
"path": "/home/",
"remote_type": "sftp"
},
"sftp_vm_webservices_src_etc":{
"description": "sftp server for webservices",
"hostname": "5.199.136.22",
"username": "nonroot",
"port": "22",
"path": "/etc/",
"remote_type": "sftp"
},
"sftp_vm_webservices_src_var":{
"description": "sftp server for webservices",
"hostname": "5.199.136.22",
"username": "nonroot",
"port": "22",
"path": "/var/",
"remote_type": "sftp"
},
"sftp_vm_webservices_src_mysqldump":{
"description": "sftp server for webservices",
"hostname": "5.199.136.22",
"username": "nonroot",
"port": "22",
"path": "",
"remote_type": "sftp"
},
"sftp_vm_discord_autism_bot_src_etc":{
"description": "sftp server for autismbot Discord",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/etc/",
"remote_type": "sftp"
},
"sftp_vm_discord_autism_bot_src_var":{
"description": "sftp server for autismbot Discord",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/var/",
"remote_type": "sftp"
},
"sftp_vm_discord_autism_bot_src_mysqldump":{
"description": "sftp server for autismbot Discord",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "",
"remote_type": "sftp"
},
"sftp_vm_discord_autism_bot_src_autism_bot":{
"description": "sftp server for autismbot Discord",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/autism_bot/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_ovh":{
"description": "sftp server for backups, fastdl & demos",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/ovh/",
"remote_type": "sftp"
},
"sftp_vm_backups2_dest_ovh":{
"description": "Second sftp server for backups, fastdl & demos",
"hostname": "78.31.67.70",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/compressed_backups/ovh/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_discord_autism_bot":{
"description": "sftp server for discord autism bot",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/autism_bot/",
"remote_type": "sftp"
},
"sftp_vm_backups2_dest_discord_autism_bot":{
"description": "Second sftp server for discord autism bot",
"hostname": "78.31.67.70",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/compressed_backups/autism_bot/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_hlstats":{
"description": "sftp server for backups, fastdl & demos",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/hlstats/",
"remote_type": "sftp"
},
"sftp_vm_backups2_dest_hlstats":{
"description": "Second sftp server for backups, fastdl & demos",
"hostname": "78.31.67.70",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/compressed_backups/hlstats/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_xenforo":{
"description": "sftp server for backups, fastdl & demos",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/xenforo/",
"remote_type": "sftp"
},
"sftp_vm_backups2_dest_xenforo":{
"description": "Second sftp server for backups, fastdl & demos",
"hostname": "78.31.67.70",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/compressed_backups/xenforo/",
"remote_type": "sftp"
},
"sftp_vm_backups_dest_webservices":{
"description": "sftp server for backups, fastdl & demos",
"hostname": "5.199.136.34",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/webservices/",
"remote_type": "sftp"
},
"sftp_vm_backups2_dest_webservices":{
"description": "Second sftp server for backups, fastdl & demos",
"hostname": "78.31.67.70",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/compressed_backups/webservices/",
"remote_type": "sftp"
}
"sftp_hetzner_src_home":{
"description": "sftp server hetzner",
"hostname": "iphere",
"username": "nonroot",
"port": "22",
"path": "/home/",
"remote_type": "sftp"
},
"sftp_hetzner_src_mysqldump":{
"description": "sftp server hetzner",
"hostname": "iphere",
"username": "nonroot",
"port": "22",
"path": "",
"remote_type": "sftp"
},
"sftp_dest_hetzner":{
"description": "sftp server hetzner",
"hostname": "iphere",
"username": "nonroot",
"port": "22",
"path": "/home/nonroot/backups/",
"remote_type": "sftp"
},
"sftp_dest_backups2":{
"description": "Second sftp server",
"hostname": "iphere",
"username": "autismbot5",
"port": "22",
"path": "/home/autismbot5/backups/",
"remote_type": "sftp"
}
},
"jobs":[
{
"job_name": "backup_remote_webservices_etc",
"job_description": "zips the remote webservices etc folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_webservices_src_etc",
"dest": "sftp_vm_backups_dest_webservices",
"dest2": "sftp_vm_backups2_dest_webservices",
"zipname":"webservices_etc_backup",
"download_dir": "/home/file_mover/"
},
"job_name": "backup_remote_hetzner_home",
"job_description": "zips hetzner home directory",
"src": "sftp_hetzner_src_home",
"dest": "sftp_dest_hetzner",
"dest2": "sftp_dest_backups2",
"zipname":"hetzner_home_dir",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_webservices_var",
"job_description": "zips the remote webservices var folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_webservices_src_var",
"dest": "sftp_vm_backups_dest_webservices",
"dest2": "sftp_vm_backups2_dest_webservices",
"zipname":"webservices_var_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_webservices_home",
"job_description": "zips the remote webservices home folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_webservices_src_home",
"dest": "sftp_vm_backups_dest_webservices",
"dest2": "sftp_vm_backups2_dest_webservices",
"zipname":"webservices_home_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_webservices_mysqldump",
"job_description": "dumps the remote webservices vm databases and moves them to backup/fastdl/demo vm",
"src": "sftp_vm_webservices_src_mysqldump",
"dest": "sftp_vm_backups_dest_webservices",
"dest2": "sftp_vm_backups2_dest_webservices",
"zipname":"webservices_mysqldump_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_xenforo_etc",
"job_description": "zips the remote xenforo etc folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_xenforo_src_etc",
"dest": "sftp_vm_backups_dest_xenforo",
"dest2": "sftp_vm_backups2_dest_xenforo",
"zipname":"xenforo_etc_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_xenforo_var",
"job_description": "zips the remote xenforo var folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_xenforo_src_var",
"dest": "sftp_vm_backups_dest_xenforo",
"dest2": "sftp_vm_backups2_dest_xenforo",
"zipname":"xenforo_var_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_xenforo_home",
"job_description": "zips the remote xenforo home folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_xenforo_src_home",
"dest": "sftp_vm_backups_dest_xenforo",
"dest2": "sftp_vm_backups2_dest_xenforo",
"zipname":"xenforo_home_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_xenforo_mysqldump",
"job_description": "dumps the remote xenforo vm databases and moves them to backup/fastdl/demo vm",
"src": "sftp_vm_xenforo_src_mysqldump",
"dest": "sftp_vm_backups_dest_xenforo",
"dest2": "sftp_vm_backups2_dest_xenforo",
"zipname":"xenforo_mysqldump_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_hlstats_etc",
"job_description": "zips the remote hlstats etc folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_hlstats_src_etc",
"dest": "sftp_vm_backups_dest_hlstats",
"dest2": "sftp_vm_backups2_dest_hlstats",
"zipname":"hlstats_etc_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_hlstats_var",
"job_description": "zips the remote hlstats var folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_hlstats_src_var",
"dest": "sftp_vm_backups_dest_hlstats",
"dest2": "sftp_vm_backups2_dest_hlstats",
"zipname":"hlstats_var_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_hlstats_mysqldump",
"job_description": "dumps the remote hlstats vm databases and moves them to backup/fastdl/demo vm",
"src": "sftp_vm_hlstats_src_mysqldump",
"dest": "sftp_vm_backups_dest_hlstats",
"dest2": "sftp_vm_backups2_dest_hlstats",
"zipname":"hlstats_mysqldump_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_hlstats_home",
"job_description": "zips the remote hlstats home folder and moves it to backup/fastdl/demo vm",
"src": "sftp_vm_hlstats_src_home",
"dest": "sftp_vm_backups_dest_hlstats",
"dest2": "sftp_vm_backups2_dest_hlstats",
"zipname":"hlstats_home_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_autism_bot_etc",
"job_description": "zips the remote discord autism bot etc folder and moves it to discord autism bot vm",
"src": "sftp_vm_discord_autism_bot_src_etc",
"dest": "sftp_vm_backups_dest_discord_autism_bot",
"dest2": "sftp_vm_backups2_dest_discord_autism_bot",
"zipname":"discord_autism_bot_etc_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_autism_bot_var",
"job_description": "zips the remote discord autism bot var folder and moves it to discord autism bot vm",
"src": "sftp_vm_discord_autism_bot_src_var",
"dest": "sftp_vm_backups_dest_discord_autism_bot",
"dest2": "sftp_vm_backups2_dest_discord_autism_bot",
"zipname":"discord_autism_bot_var_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_autism_bot_mysqldump",
"job_description": "dumps the remote discord autism bot vm databases and moves them to discord autism bot vm",
"src": "sftp_vm_discord_autism_bot_src_mysqldump",
"dest": "sftp_vm_backups_dest_discord_autism_bot",
"dest2": "sftp_vm_backups2_dest_discord_autism_bot",
"zipname":"discord_autism_bot_mysqldump_backup",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_remote_autism_bot_discord",
"job_description": "zips the remote autism bot discord autism bot discord folder and moves it to backup/demo vm",
"src": "sftp_vm_discord_autism_bot_src_autism_bot",
"dest": "sftp_vm_backups_dest_discord_autism_bot",
"dest2": "sftp_vm_backups2_dest_discord_autism_bot",
"zipname":"discord_autism_bot_folder_backup",
"download_dir": "/home/file_mover/"
},
"job_name": "hetzner_mysqldump",
"job_description": "dumps hetzner databases",
"src": "sftp_hetzner_src_mysqldump",
"dest": "sftp_dest_hetzner",
"dest2": "sftp_dest_backups2",
"zipname":"hetzner_mysqldump",
"download_dir": "/home/file_mover/"
},
{
"job_name": "backup_local_gameservers",
"job_description": "zips the local gameservers folder and moves it to backup/fastdl/demo vm",
"job_description": "zips the local gameservers folder",
"src": "local_dir_gameservers",
"dest": "sftp_vm_backups_dest_ovh",
"dest2": "sftp_vm_backups2_dest_ovh",
"zipname":"ovh_gameservers_backup"
},
{
"job_name": "backup_local_var",
"job_description": "zips the local var folder and moves it to backup/fastdl/demo vm",
"src": "local_dir_var",
"dest": "sftp_vm_backups_dest_ovh",
"dest2": "sftp_vm_backups2_dest_ovh",
"zipname":"ovh_var_backup"
},
{
"job_name": "backup_local_etc",
"job_description": "zips the local etc folder and moves it to backup/fastdl/demo vm",
"src": "local_dir_etc",
"dest": "sftp_vm_backups_dest_ovh",
"dest2": "sftp_vm_backups2_dest_ovh",
"zipname":"ovh_etc_backup"
},
{
"job_name": "backup_local_mysqldump",
"job_description": "zips the local databases and moves it to backup/fastdl/demo vm",
"src": "local_dir_mysqldump",
"dest": "sftp_vm_backups_dest_ovh",
"dest2": "sftp_vm_backups2_dest_ovh",
"zipname":"ovh_mysqldump_backup"
}
"dest": "sftp_dest_hetzner",
"dest2": "sftp_dest_backups2",
"zipname":"ovh_gameservers"
}
],
"settings": {
"sftp": {
"remote_attempts": "5",
"remote_delay": "15",
"5.199.136.34":{
"username": {
"password": ""
}
},
"5.199.136.22":{
"username": {
"password": ""
}
},
"5.104.107.130":{
"username": {
"password": ""
}
},
"5.104.107.39":{
"username": {
"password": ""
}
},
"78.31.67.70": {
"username": {
"iphere":{
"nonroot": {
"password": ""
}
},
"iphere": {
"autismbot5": {
"password": ""
}
}

View File

@ -45,8 +45,8 @@ def load_config(config_file):
sys.exit(1)
def main():
#runs demos every 5 minutes but backups thrice a week with systemctl services/timers
config_file = 'config_demos.json'
#runs backups
config_file = 'config_backups.json'
if sys.argv[1:]:
config_file = sys.argv[1]
remotes, jobs, settings = load_config(config_file)
@ -56,15 +56,9 @@ def main():
dest2 = create_remote(remotes[job["dest2"]], settings)
if index == 0:
#delete old files (not very adaptable, cant handle dirs in dirs)
dest.delete_remote_zips("/home/nonroot/backups")
dest2.delete_remote_zips("/home/autismbot5/compressed_backups")
if 'demo_' in job["job_name"]:
source_files = src.list_dir()
distribute_files(source_files, dest)
for pathfile in source_files:
src.delete_local_demo(pathfile)
dest.delete_remote_demos(source_files)
elif "backup_local" in job["job_name"]:
dest.delete_remote_zips("/home/nonroot/backups/")
dest2.delete_remote_zips("/home/autismbot5/backups/")
if "backup_local" in job["job_name"]:
zip_file_path = src.zip_local_directory(job)
if not dest.put(zip_file_path, dest.path):
log_msg = ''.join(["failed putting file: ", str(pathfile)])
@ -120,3 +114,4 @@ def main():
if __name__ == '__main__':
main()

View File

@ -1,7 +1,3 @@
from pathlib import Path
import contextlib
import os
from datetime import timedelta, datetime
import subprocess
class local_dir_remote:

View File

@ -50,11 +50,12 @@ class sftp_remote:
channel = self.ssh.get_transport().open_session(timeout=120)
#-T3= 3 threads, use 70% of available ram
#TODO exclude dir where hlstats spams pictures of the player graphs
if 'tar_recursive' == command_state:
exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar --exclude={zipname} -cJf {zipname} {self.path}"
exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar --exclude={zipname} --exclude=/home/nonroot/backups/ --exclude=/home/nonroot/demos --exclude=/home/nonroot/snap --exclude=logs/ --ignore-failed-read -cJf {zipname} {self.path}"
elif 'mysqldump' == command_state:
#check README for my.cnf regarding how this works
exec_command_state = f'mysqldump -u root --all-databases > mysqldump.sql'
exec_command_state = f'mysqldump -u backups_db --all-databases > mysqldump.sql'
else: #tar_mysqldump
exec_command_state = f"XZ_OPT='-T3 --memlimit-compress=70%' tar -cJf {zipname} /home/{self.username}/mysqldump.sql"
exec_cmd = f'cd /home/{self.username}/; {exec_command_state}'
@ -187,14 +188,13 @@ class sftp_remote:
def delete_remote_zips(self, backups_dir):
self.connect()
for directories in self.sftp.listdir(backups_dir):
for files in self.sftp.listdir(f'{backups_dir}/{directories}'):
pathfile = f'{backups_dir}/{directories}/{files}'
if not pathfile.endswith(".xz"):
continue
utime = self.sftp.stat(pathfile).st_mtime
last_modified = datetime.fromtimestamp(utime)
self.delete_file(last_modified, pathfile, 25)
for files in self.sftp.listdir(f'{backups_dir}'):
pathfile = f'{backups_dir}{files}'
if not pathfile.endswith(".xz"):
continue
utime = self.sftp.stat(pathfile).st_mtime
last_modified = datetime.fromtimestamp(utime)
self.delete_file(last_modified, pathfile, 40)
self.disconnect()
def delete_remote_zip_temp(self, zipname):