added support for remote jobs to zip /etc/, /var/ and home directories. just missing mysqldumps now
This commit is contained in:
christian
parent
b3cbd7f6b6
commit
30f4228f24
@ -15,6 +15,118 @@
|
||||
"path": "/etc/",
|
||||
"remote_type": "local_dir"
|
||||
},
|
||||
"sftp_vm_hlstats_src_home":{
|
||||
"description": "sftp server for hlstats",
|
||||
"hostname": "51.15.159.31",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "/home/",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_hlstats_src_etc":{
|
||||
"description": "sftp server for hlstats",
|
||||
"hostname": "51.15.159.31",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "/etc/",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_hlstats_src_var":{
|
||||
"description": "sftp server for hlstats",
|
||||
"hostname": "51.15.159.31",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "/var/",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_hlstats_src_mysqldump":{
|
||||
"description": "sftp server for hlstats",
|
||||
"hostname": "51.15.159.31",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_xenforo_src_home":{
|
||||
"description": "sftp server for xenforo",
|
||||
"hostname": "163.172.225.175",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "/home/",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_xenforo_src_etc":{
|
||||
"description": "sftp server for xenforo",
|
||||
"hostname": "163.172.225.175",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "/etc/",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_xenforo_src_var":{
|
||||
"description": "sftp server for xenforo",
|
||||
"hostname": "163.172.225.175",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "/var/",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_xenforo_src_mysqldump":{
|
||||
"description": "sftp server for xenforo",
|
||||
"hostname": "163.172.225.175",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_webservices_src_home":{
|
||||
"description": "sftp server for webservices",
|
||||
"hostname": "163.172.117.46",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "/home/",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_webservices_src_etc":{
|
||||
"description": "sftp server for webservices",
|
||||
"hostname": "163.172.117.46",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "/etc/",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_webservices_src_var":{
|
||||
"description": "sftp server for webservices",
|
||||
"hostname": "163.172.117.46",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "/var/",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_webservices_src_mysqldump":{
|
||||
"description": "sftp server for webservices",
|
||||
"hostname": "163.172.117.46",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_backups_src_etc":{
|
||||
"description": "sftp server for backups/fastdl/demos",
|
||||
"hostname": "163.172.119.53",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "/etc/",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_backups_src_var":{
|
||||
"description": "sftp server for backups/fastdl/demos",
|
||||
"hostname": "163.172.119.53",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "/var/",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_backups_dest_ovh":{
|
||||
"description": "sftp server for backups, fastdl & demos",
|
||||
"hostname": "163.172.119.53",
|
||||
@ -22,9 +134,150 @@
|
||||
"port": "22",
|
||||
"path": "/home/nonroot/backups/ovh/",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_backups_dest_fastdl":{
|
||||
"description": "sftp server for backups, fastdl & demos",
|
||||
"hostname": "163.172.119.53",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "/home/nonroot/backups/fastdl/",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_backups_dest_hlstats":{
|
||||
"description": "sftp server for backups, fastdl & demos",
|
||||
"hostname": "163.172.119.53",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "/home/nonroot/backups/hlstats/",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_backups_dest_xenforo":{
|
||||
"description": "sftp server for backups, fastdl & demos",
|
||||
"hostname": "163.172.119.53",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "/home/nonroot/backups/xenforo/",
|
||||
"remote_type": "sftp"
|
||||
},
|
||||
"sftp_vm_backups_dest_webservices":{
|
||||
"description": "sftp server for backups, fastdl & demos",
|
||||
"hostname": "163.172.119.53",
|
||||
"username": "nonroot",
|
||||
"port": "22",
|
||||
"path": "/home/nonroot/backups/webservices/",
|
||||
"remote_type": "sftp"
|
||||
}
|
||||
},
|
||||
"jobs":[
|
||||
{
|
||||
"job_name": "backup_remote_webservices_etc",
|
||||
"job_description": "zips the remote webservices etc folder and moves it to backup/fastdl/demo vm",
|
||||
"src": "sftp_vm_webservices_src_etc",
|
||||
"dest": "sftp_vm_backups_dest_webservices",
|
||||
"zipname":"webservices_etc_backup",
|
||||
"download_dir": "/home/file_mover/"
|
||||
},
|
||||
{
|
||||
"job_name": "backup_remote_webservices_var",
|
||||
"job_description": "zips the remote webservices var folder and moves it to backup/fastdl/demo vm",
|
||||
"src": "sftp_vm_webservices_src_var",
|
||||
"dest": "sftp_vm_backups_dest_webservices",
|
||||
"zipname":"webservices_var_backup",
|
||||
"download_dir": "/home/file_mover/"
|
||||
},
|
||||
{
|
||||
"job_name": "backup_remote_webservices_home",
|
||||
"job_description": "zips the remote webservices home folder and moves it to backup/fastdl/demo vm",
|
||||
"src": "sftp_vm_webservices_src_home",
|
||||
"dest": "sftp_vm_backups_dest_webservices",
|
||||
"zipname":"webservices_home_backup",
|
||||
"download_dir": "/home/file_mover/"
|
||||
},
|
||||
{
|
||||
"job_name": "backup_remote_webservices_mysqldump",
|
||||
"job_description": "dumps the remote webservices vm databases and moves them to backup/fastdl/demo vm",
|
||||
"src": "sftp_vm_webservices_src_mysqldump",
|
||||
"dest": "sftp_vm_backups_dest_webservices",
|
||||
"zipname":"webservices_mysqldump_backup"
|
||||
},
|
||||
{
|
||||
"job_name": "backup_remote_xenforo_etc",
|
||||
"job_description": "zips the remote xenforo etc folder and moves it to backup/fastdl/demo vm",
|
||||
"src": "sftp_vm_xenforo_src_etc",
|
||||
"dest": "sftp_vm_backups_dest_xenforo",
|
||||
"zipname":"xenforo_etc_backup",
|
||||
"download_dir": "/home/file_mover/"
|
||||
},
|
||||
{
|
||||
"job_name": "backup_remote_xenforo_var",
|
||||
"job_description": "zips the remote xenforo var folder and moves it to backup/fastdl/demo vm",
|
||||
"src": "sftp_vm_xenforo_src_var",
|
||||
"dest": "sftp_vm_backups_dest_xenforo",
|
||||
"zipname":"xenforo_var_backup",
|
||||
"download_dir": "/home/file_mover/"
|
||||
},
|
||||
{
|
||||
"job_name": "backup_remote_xenforo_home",
|
||||
"job_description": "zips the remote xenforo home folder and moves it to backup/fastdl/demo vm",
|
||||
"src": "sftp_vm_xenforo_src_home",
|
||||
"dest": "sftp_vm_backups_dest_xenforo",
|
||||
"zipname":"xenforo_home_backup",
|
||||
"download_dir": "/home/file_mover/"
|
||||
},
|
||||
{
|
||||
"job_name": "backup_remote_xenforo_mysqldump",
|
||||
"job_description": "dumps the remote xenforo vm databases and moves them to backup/fastdl/demo vm",
|
||||
"src": "sftp_vm_xenforo_src_mysqldump",
|
||||
"dest": "sftp_vm_backups_dest_xenforo",
|
||||
"zipname":"xenforo_mysqldump_backup"
|
||||
},
|
||||
{
|
||||
"job_name": "backup_remote_hlstats_etc",
|
||||
"job_description": "zips the remote hlstats etc folder and moves it to backup/fastdl/demo vm",
|
||||
"src": "sftp_vm_hlstats_src_etc",
|
||||
"dest": "sftp_vm_backups_dest_hlstats",
|
||||
"zipname":"hlstats_etc_backup",
|
||||
"download_dir": "/home/file_mover/"
|
||||
},
|
||||
{
|
||||
"job_name": "backup_remote_hlstats_var",
|
||||
"job_description": "zips the remote hlstats var folder and moves it to backup/fastdl/demo vm",
|
||||
"src": "sftp_vm_hlstats_src_var",
|
||||
"dest": "sftp_vm_backups_dest_hlstats",
|
||||
"zipname":"hlstats_var_backup",
|
||||
"download_dir": "/home/file_mover/"
|
||||
},
|
||||
{
|
||||
"job_name": "backup_remote_hlstats_mysqldump",
|
||||
"job_description": "dumps the remote hlstats vm databases and moves them to backup/fastdl/demo vm",
|
||||
"src": "sftp_vm_hlstats_src_mysqldump",
|
||||
"dest": "sftp_vm_backups_dest_hlstats",
|
||||
"zipname":"hlstats_mysqldump_backup"
|
||||
},
|
||||
{
|
||||
"job_name": "backup_remote_hlstats_home",
|
||||
"job_description": "zips the remote hlstats home folder and moves it to backup/fastdl/demo vm",
|
||||
"src": "sftp_vm_hlstats_src_home",
|
||||
"dest": "sftp_vm_backups_dest_hlstats",
|
||||
"zipname":"hlstats_home_backup",
|
||||
"download_dir": "/home/file_mover/"
|
||||
},
|
||||
{
|
||||
"job_name": "backup_remote_fastdl_etc",
|
||||
"job_description": "zips the remote backups/fastdl/demos etc folder and moves it to backup/fastdl/demo vm",
|
||||
"src": "sftp_vm_backups_src_etc",
|
||||
"dest": "sftp_vm_backups_dest_fastdl",
|
||||
"zipname":"fastdl_etc_backup",
|
||||
"download_dir": "/home/file_mover/"
|
||||
},
|
||||
{
|
||||
"job_name": "backup_remote_fastdl_var",
|
||||
"job_description": "zips the remote backups/fastdl/demos var folder and moves it to backup/fastdl/demo vm",
|
||||
"src": "sftp_vm_backups_src_var",
|
||||
"dest": "sftp_vm_backups_dest_fastdl",
|
||||
"zipname":"fastdl_var_backup",
|
||||
"download_dir": "/home/file_mover/"
|
||||
},
|
||||
{
|
||||
"job_name": "backup_local_gameservers",
|
||||
"job_description": "zips the local gameservers folder and moves it to backup/fastdl/demo vm",
|
||||
@ -51,11 +304,26 @@
|
||||
"sftp": {
|
||||
"remote_attempts": "5",
|
||||
"remote_delay": "15",
|
||||
"ip address":{
|
||||
"ip":{
|
||||
"username": {
|
||||
"password": "password"
|
||||
}
|
||||
}
|
||||
},
|
||||
"ip":{
|
||||
"username": {
|
||||
"password": "password"
|
||||
}
|
||||
},
|
||||
"ip":{
|
||||
"username": {
|
||||
"password": "password"
|
||||
}
|
||||
},
|
||||
"ip":{
|
||||
"password": {
|
||||
"password": "password"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -67,7 +67,24 @@ def main():
|
||||
sys.exit(1)
|
||||
src.delete_local_zip(zip_file_path)
|
||||
dest.delete_remote_zips(zip_file_path)
|
||||
|
||||
elif "mysqldump" in job["job_name"]:
|
||||
pass
|
||||
elif "backup_remote" in job["job_name"]:
|
||||
zipname = src.zip_remote_directory(job)
|
||||
if zipname is None:
|
||||
logging.warning(f'failed zipping remote directory at job {job}')
|
||||
sys.exit(1)
|
||||
local_zip_path_name = src.get_remote_files(job, zipname)
|
||||
if local_zip_path_name is None:
|
||||
logging.warning(f'failed getting remote zip at job {job}')
|
||||
src.delete_remote_zip_temp(zipname)
|
||||
sys.exit(1)
|
||||
src.delete_remote_zip_temp(zipname)
|
||||
if not dest.put(local_zip_path_name, dest.path):
|
||||
logging.warning(f'failed putting local zip {local_zip_path_name} at job {job}')
|
||||
src.delete_local_zip(local_zip_path_name)
|
||||
sys.exit(1)
|
||||
src.delete_local_zip(local_zip_path_name)
|
||||
logging.info(('finished job: ', job))
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
import paramiko
|
||||
from paramiko.ssh_exception import SSHException, NoValidConnectionsError
|
||||
import time
|
||||
import sys
|
||||
import hashlib
|
||||
import stat
|
||||
from datetime import datetime, timedelta
|
||||
@ -20,6 +19,7 @@ class sftp_remote:
|
||||
self.remote_delay = settings[self.remote_type]['remote_delay']
|
||||
self.remote_error = None
|
||||
self.password = settings[self.remote_type][self.hostname][self.username]['password']
|
||||
self.ssh = None
|
||||
|
||||
def connect(self):
|
||||
self.transport = paramiko.Transport((self.hostname, int(self.port)))
|
||||
@ -34,6 +34,64 @@ class sftp_remote:
|
||||
del (self.sftp)
|
||||
self.transport.close()
|
||||
|
||||
def ssh_connect(self):
|
||||
self.ssh = paramiko.SSHClient()
|
||||
#i trust my vm's
|
||||
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
self.ssh.connect(self.hostname, username=self.username, password=self.password)
|
||||
|
||||
def ssh_disconnect(self):
|
||||
self.ssh.close()
|
||||
|
||||
def zip_remote_directory(self, job):
|
||||
zipname = f'{job["zipname"]}-{str(datetime.now()).split(" ")[0]}.zip'
|
||||
self.ssh_connect()
|
||||
channel = self.ssh.get_transport().open_session(timeout=120)
|
||||
exec_cmd = f'cd /home/{self.username}/; zip -r {zipname} {self.path}'
|
||||
keyboard_interrupt = False
|
||||
try:
|
||||
channel.exec_command(exec_cmd)
|
||||
#2^21 (2,097,152) characters before filling up the buffers of exec_command
|
||||
while True:
|
||||
buf = channel.recv(1024)
|
||||
if not buf:
|
||||
break
|
||||
#priting the output for sure causes it to run a bit slower
|
||||
print('buffer: ', buf)
|
||||
channel.recv_exit_status()
|
||||
except KeyboardInterrupt:
|
||||
keyboard_interrupt = True
|
||||
finally:
|
||||
channel.close()
|
||||
self.ssh_disconnect()
|
||||
if keyboard_interrupt:
|
||||
print("manually interrupted")
|
||||
sys.exit(1)
|
||||
return zipname
|
||||
|
||||
def get_remote_files(self, job, zipname):
|
||||
total_attempts = int(self.remote_attempts)
|
||||
while total_attempts > 0:
|
||||
self.connect()
|
||||
job_path = job["download_dir"]
|
||||
self.sftp.get(f'/home/{self.username}/{zipname}', f'{job_path}{zipname}')
|
||||
sha256_first = self.digest(f'{job_path}{zipname}')
|
||||
os.remove(f'{job_path}{zipname}')
|
||||
self.disconnect()
|
||||
|
||||
#redownloading to validate the SHA sum
|
||||
self.connect()
|
||||
self.sftp.get(f'/home/{self.username}/{zipname}', f'{job_path}{zipname}')
|
||||
sha256_second = self.digest(f'{job_path}{zipname}')
|
||||
self.disconnect()
|
||||
if sha256_first == sha256_second:
|
||||
return f'{job_path}{zipname}'
|
||||
os.remove(f'{job_path}{zipname}')
|
||||
total_attempts = self.subtract_remote_attempts(total_attempts)
|
||||
if total_attempts == 0:
|
||||
return None
|
||||
|
||||
|
||||
def subtract_remote_attempts(self, total_attempts):
|
||||
time.sleep(self.remote_delay)
|
||||
return total_attempts -1
|
||||
@ -115,3 +173,13 @@ class sftp_remote:
|
||||
pathfile, last_modified = self.change_path(zip_file_path)
|
||||
self.delete_file(last_modified, pathfile, 10)
|
||||
self.disconnect()
|
||||
|
||||
def delete_remote_zip_temp(self, zipname):
|
||||
self.connect()
|
||||
self.sftp.remove(f'/home/{self.username}/{zipname}')
|
||||
self.disconnect()
|
||||
|
||||
def delete_local_zip(self, local_zip_path_name):
|
||||
os.remove(local_zip_path_name)
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user