From b8314469847200c5f1091fa6a354332506aedcb0 Mon Sep 17 00:00:00 2001 From: Christian Date: Sat, 13 Feb 2021 21:27:46 +0100 Subject: [PATCH] fixed issue related to /home partition having all space and the /var/www folder running out of space --- demo_mover/README.md | 8 +---- demo_mover/config.json | 2 +- demo_mover/demo_mover.py | 20 +++++------ demo_mover/remote_local_dir.py | 5 +-- demo_mover/remote_sftp.py | 49 ++++++++------------------- demo_mover/requirements.txt | 0 demo_mover/run_demo_mover.sh | 0 demo_mover/systemd/demo_mover.service | 3 +- demo_mover/systemd/demo_mover.timer | 4 +-- 9 files changed, 31 insertions(+), 60 deletions(-) mode change 100644 => 100755 demo_mover/README.md mode change 100644 => 100755 demo_mover/config.json mode change 100644 => 100755 demo_mover/demo_mover.py mode change 100644 => 100755 demo_mover/remote_local_dir.py mode change 100644 => 100755 demo_mover/remote_sftp.py mode change 100644 => 100755 demo_mover/requirements.txt mode change 100644 => 100755 demo_mover/run_demo_mover.sh mode change 100644 => 100755 demo_mover/systemd/demo_mover.service mode change 100644 => 100755 demo_mover/systemd/demo_mover.timer diff --git a/demo_mover/README.md b/demo_mover/README.md old mode 100644 new mode 100755 index 9bffde39..dc2def79 --- a/demo_mover/README.md +++ b/demo_mover/README.md @@ -8,10 +8,4 @@ ##config file: server specifics -##python3 demo_mover.py config.json - -##systemctl enable demo_mover.timer -##systemctl start demo_mover.timer -##logging: journalctl -f -u demo_mover.service -##logging: systemctl status demo_mover.service -###test commit on new git +##nohup screen -d -m -S demo_mover python3 demo_mover.py config.json diff --git a/demo_mover/config.json b/demo_mover/config.json old mode 100644 new mode 100755 index e3e90daf..41fe8464 --- a/demo_mover/config.json +++ b/demo_mover/config.json @@ -15,7 +15,7 @@ "hostname": "163.172.117.46", "username": "gameservers", "port": "22", - "path": "/var/www/demos/", + "path": "/home/gameservers/demos/", "remote_type": "sftp" } }, diff --git a/demo_mover/demo_mover.py b/demo_mover/demo_mover.py old mode 100644 new mode 100755 index 48c44349..a9b2fa43 --- a/demo_mover/demo_mover.py +++ b/demo_mover/demo_mover.py @@ -50,18 +50,14 @@ def main(): config_file = sys.argv[1] remotes, jobs, settings = load_config(config_file) for job in jobs: - try: - src = create_remote(remotes[job["src"]], settings) - dest = create_remote(remotes[job["dest"]], settings) - source_files = src.list_dir() - distribute_files(source_files, dest) - except Exception as e: - logging.warning('exception caught: ', exc_info = True) - finally: - for pathfile in source_files: - src.delete_local(pathfile) - dest.delete_remote(source_files) - logging.info('finished jobs') + src = create_remote(remotes[job["src"]], settings) + dest = create_remote(remotes[job["dest"]], settings) + source_files = src.list_dir() + distribute_files(source_files, dest) + for pathfile in source_files: + src.delete_local(pathfile) + dest.delete_remote(source_files) + logging.info(('finished job: ', job)) if __name__ == '__main__': main() diff --git a/demo_mover/remote_local_dir.py b/demo_mover/remote_local_dir.py old mode 100644 new mode 100755 index f5abbd5b..b9dc5acc --- a/demo_mover/remote_local_dir.py +++ b/demo_mover/remote_local_dir.py @@ -1,6 +1,7 @@ from pathlib import Path import contextlib import os +from datetime import timedelta, datetime class local_dir_remote: def __init__(self, config): @@ -10,12 +11,12 @@ class local_dir_remote: def list_dir(self): path = Path(self.path) file_list = [] - delta = timedelta(minutes=5) + delta = timedelta(minutes=2) for file_demo in list(path.glob('*')): mtime = datetime.fromtimestamp(os.stat(file_demo).st_mtime) if mtime < datetime.now() - delta: file_list.append(file_demo) - return file_list + return file_list def delete_local(self, pathfile): with contextlib.suppress(FileNotFoundError): diff --git a/demo_mover/remote_sftp.py b/demo_mover/remote_sftp.py old mode 100644 new mode 100755 index 71026801..a097fded --- a/demo_mover/remote_sftp.py +++ b/demo_mover/remote_sftp.py @@ -22,19 +22,9 @@ class sftp_remote: self.password = settings[self.remote_type][self.hostname][self.username]['password'] def connect(self): - total_attempts = int(self.remote_attempts) self.transport = paramiko.Transport((self.hostname, int(self.port))) - while total_attempts > 0: - try: - self.transport.connect(username = self.username, password = self.password) - self.sftp = paramiko.SFTPClient.from_transport(self.transport) - return True - except NoValidConnectionsError as e: - print('sftp connect failed: ', e) - total_attempts = self.subtract_remote_attempts(total_attempts) - if total_attempts == 0: - self.remote_error = e - return False + self.transport.connect(username = self.username, password = self.password) + self.sftp = paramiko.SFTPClient.from_transport(self.transport) def disconnect(self): del (self.sftp) @@ -81,29 +71,18 @@ class sftp_remote: filename = local_path_str.split("/")[-1] remote_path = remote_path + filename local_temp_folder = local_temp_folder + filename - #print('local_path_str: ', local_path_str, '\n remote_path: ', remote_path, '\n local_temp_folder: ', local_temp_folder) - while total_attempts > 0: - try: - self.connect() - self.sftp.put(local_path, remote_path) - self.disconnect() - self.connect() - self.sftp.get(remote_path, local_temp_folder) - self.disconnect() - local_sha256 = self.digest(local_temp_folder) - os.remove(local_temp_folder) - #print('removing: ', local_temp_folder) - time.sleep(2) - sha256_local_upload_check = self.digest(local_path) - if sha256 != sha256_local_upload_check: - return False #file is currently being uploaded - if local_sha256 == sha256: - #print('sha confirmed') - return True - except SSHException as e: - print(e) - self.remote_error = e + self.connect() + self.sftp.put(local_path, remote_path) + self.disconnect() + self.connect() + self.sftp.get(remote_path, local_temp_folder) + self.disconnect() + local_sha256 = self.digest(local_temp_folder) + os.remove(local_temp_folder) + if local_sha256 == sha256: + #print('sha confirmed') + return True total_attempts = self.subtract_remote_attempts(total_attempts) if total_attempts == 0: return False @@ -119,7 +98,7 @@ class sftp_remote: pathfile = self.path + filename utime = self.sftp.stat(pathfile).st_mtime last_modified = datetime.fromtimestamp(utime) - if (datetime.now() - last_modified) > timedelta(days=30 * 3): + if (datetime.now() - last_modified) > timedelta(days=30 * 1.5): print('deleting file remotely: ', pathfile) self.sftp.remove(pathfile) self.disconnect() diff --git a/demo_mover/requirements.txt b/demo_mover/requirements.txt old mode 100644 new mode 100755 diff --git a/demo_mover/run_demo_mover.sh b/demo_mover/run_demo_mover.sh old mode 100644 new mode 100755 diff --git a/demo_mover/systemd/demo_mover.service b/demo_mover/systemd/demo_mover.service old mode 100644 new mode 100755 index 5a138770..07104a65 --- a/demo_mover/systemd/demo_mover.service +++ b/demo_mover/systemd/demo_mover.service @@ -1,8 +1,9 @@ [Unit] -Description=Moves demos from the local machine to remote destinations every 20 minutes +Description=Moves demos from the local machine to remote destinations every 5 minutes [Service] Type=simple User=gameservers WorkingDirectory=/home/gameservers/demo_mover ExecStart=/home/gameservers/demo_mover/run_demo_mover.sh + diff --git a/demo_mover/systemd/demo_mover.timer b/demo_mover/systemd/demo_mover.timer old mode 100644 new mode 100755 index 42fea8a4..15de12cd --- a/demo_mover/systemd/demo_mover.timer +++ b/demo_mover/systemd/demo_mover.timer @@ -1,9 +1,9 @@ [Unit] -Description=Moves demos from the local machine to remote destinations every 20 minutes +Description=Moves demos from the local machine to remote destinations every 5 minutes Requires=demo_mover.service [Timer] -OnCalendar=*-*-* *:0,20,40 +OnCalendar=*-*-* *:0,5,10,15,20,25,30,35,40,45,50,55 [Install] WantedBy=multi-user.target