removed old directory
This commit is contained in:
christian
parent
d889ee06ea
commit
14c80a36b0
@ -1,22 +0,0 @@
|
||||
#simple demo file mover by jenz
|
||||
|
||||
##virtualenv venv
|
||||
|
||||
##source venv/bin/activate
|
||||
|
||||
##pip3 install -r requirements.txt
|
||||
|
||||
##config file: server specifics
|
||||
|
||||
|
||||
##python3 demo_mover.py config.json
|
||||
|
||||
##systemctl enable demo_mover.timer
|
||||
|
||||
##systemctl start demo_mover.timer
|
||||
|
||||
##logging: journalctl -f -u demo_mover.service
|
||||
|
||||
##logging: systemctl status demo_mover.service
|
||||
|
||||
###test commit on new git
|
||||
@ -1,47 +0,0 @@
|
||||
{
|
||||
"remotes":{
|
||||
"local_dir_demos_ze_src":{
|
||||
"description": "local machine demos ze",
|
||||
"path": "/home/gameservers/css_ze/cstrike/demos/demos/",
|
||||
"remote_type": "local_dir"
|
||||
},
|
||||
"local_dir_demos_mg_src":{
|
||||
"description": "local machine demos mg",
|
||||
"path": "/home/gameservers/css_mg/cstrike/demos/",
|
||||
"remote_type": "local_dir"
|
||||
},
|
||||
"sftp_vm_104_dest":{
|
||||
"description": "sftp server for backups, fastdl & demos",
|
||||
"hostname": "163.172.117.46",
|
||||
"username": "gameservers",
|
||||
"port": "22",
|
||||
"path": "/home/gameservers/demos/",
|
||||
"remote_type": "sftp"
|
||||
}
|
||||
},
|
||||
"jobs":[
|
||||
{
|
||||
"job_name": "demo_move_ze",
|
||||
"job_description": "move demos to vm 104",
|
||||
"src": "local_dir_demos_ze_src",
|
||||
"dest": "sftp_vm_104_dest"
|
||||
},
|
||||
{
|
||||
"job_name": "demo_move_mg",
|
||||
"job_description": "move demos to vm 104",
|
||||
"src": "local_dir_demos_mg_src",
|
||||
"dest": "sftp_vm_104_dest"
|
||||
}
|
||||
],
|
||||
"settings": {
|
||||
"sftp": {
|
||||
"remote_attempts": "5",
|
||||
"remote_delay": "15",
|
||||
"163.172.117.46":{
|
||||
"gameservers": {
|
||||
"password": "ndfybf3u24bfapbfpabfib3rIFB"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,63 +0,0 @@
|
||||
import paramiko
|
||||
import sys
|
||||
import json
|
||||
import logging
|
||||
import unicodedata
|
||||
logging.basicConfig(
|
||||
format='%(asctime)s %(levelname)-8s %(message)s',
|
||||
level=logging.INFO,
|
||||
datefmt='%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
|
||||
def create_remote(config, settings):
|
||||
type_r = config["remote_type"]
|
||||
if type_r == "sftp":
|
||||
import remote_sftp
|
||||
remote = remote_sftp.sftp_remote(config, settings)
|
||||
elif type_r == "local_dir":
|
||||
import remote_local_dir
|
||||
remote = remote_local_dir.local_dir_remote(config)
|
||||
return remote
|
||||
|
||||
def distribute_files(path_list, dest):
|
||||
for pathfile in path_list:
|
||||
if not str(pathfile).endswith('.dem'):
|
||||
continue
|
||||
log_msg = "uploading demo: " + str(pathfile)
|
||||
logging.info(log_msg)
|
||||
if not dest.put(pathfile, dest.path):
|
||||
log_msg = ''.join(["failed putting file: ", str(pathfile)])
|
||||
logging.warning(log_msg)
|
||||
sys.exit(1)
|
||||
|
||||
def load_config(config_file):
|
||||
try:
|
||||
with open(config_file) as infile:
|
||||
try:
|
||||
json_dict = json.load(infile)
|
||||
#logging.info(json_dict)
|
||||
return json_dict["remotes"], json_dict["jobs"], json_dict["settings"]
|
||||
except json.JSONDecodeError as e:
|
||||
logging.warning('exception caught: ', exc_info = True)
|
||||
sys.exit(1)
|
||||
except FileNotFoundError:
|
||||
logging.warning('exception caught: ', exc_info = True)
|
||||
sys.exit(1)
|
||||
|
||||
def main():
|
||||
config_file = 'config.json'
|
||||
if sys.argv[1:]:
|
||||
config_file = sys.argv[1]
|
||||
remotes, jobs, settings = load_config(config_file)
|
||||
for job in jobs:
|
||||
src = create_remote(remotes[job["src"]], settings)
|
||||
dest = create_remote(remotes[job["dest"]], settings)
|
||||
source_files = src.list_dir()
|
||||
distribute_files(source_files, dest)
|
||||
for pathfile in source_files:
|
||||
src.delete_local(pathfile)
|
||||
dest.delete_remote(source_files)
|
||||
logging.info(('finished job: ', job))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -1,25 +0,0 @@
|
||||
from pathlib import Path
|
||||
import contextlib
|
||||
import os
|
||||
from datetime import timedelta, datetime
|
||||
|
||||
class local_dir_remote:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.path = config["path"]
|
||||
|
||||
def list_dir(self):
|
||||
path = Path(self.path)
|
||||
file_list = []
|
||||
delta = timedelta(minutes=2)
|
||||
for file_demo in list(path.glob('*')):
|
||||
mtime = datetime.fromtimestamp(os.stat(file_demo).st_mtime)
|
||||
if mtime < datetime.now() - delta:
|
||||
file_list.append(file_demo)
|
||||
return file_list
|
||||
|
||||
def delete_local(self, pathfile):
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
if str(pathfile).endswith('.dem'):
|
||||
os.remove(pathfile)
|
||||
print('removed demo: ', pathfile)
|
||||
@ -1,104 +0,0 @@
|
||||
import paramiko
|
||||
from paramiko.ssh_exception import SSHException, NoValidConnectionsError
|
||||
import time
|
||||
import sys
|
||||
import hashlib
|
||||
import stat
|
||||
from datetime import datetime, timedelta
|
||||
import os
|
||||
|
||||
class sftp_remote:
|
||||
def __init__(self, config, settings):
|
||||
self.config = config
|
||||
self.path = config['path']
|
||||
self.description = config['description']
|
||||
self.hostname = config['hostname']
|
||||
self.username = config['username']
|
||||
self.port = config['port']
|
||||
self.remote_type = config['remote_type']
|
||||
self.remote_attempts = settings[self.remote_type]['remote_attempts']
|
||||
self.remote_delay = settings[self.remote_type]['remote_delay']
|
||||
self.remote_error = None
|
||||
self.password = settings[self.remote_type][self.hostname][self.username]['password']
|
||||
|
||||
def connect(self):
|
||||
self.transport = paramiko.Transport((self.hostname, int(self.port)))
|
||||
self.transport.connect(username = self.username, password = self.password)
|
||||
self.sftp = paramiko.SFTPClient.from_transport(self.transport)
|
||||
|
||||
def disconnect(self):
|
||||
del (self.sftp)
|
||||
self.transport.close()
|
||||
|
||||
def __close__(self):
|
||||
del (self.sftp)
|
||||
self.transport.close()
|
||||
|
||||
def subtract_remote_attempts(self, total_attempts):
|
||||
time.sleep(self.remote_delay)
|
||||
return total_attempts -1
|
||||
|
||||
def digest(self, file_path):
|
||||
hashvalue = hashlib.sha256()
|
||||
with open(file_path, 'rb') as file_:
|
||||
while True:
|
||||
chunk = file_.read(hashvalue.block_size)
|
||||
if not chunk:
|
||||
break
|
||||
hashvalue.update(chunk)
|
||||
return hashvalue.hexdigest()
|
||||
|
||||
|
||||
def put(self, local_path, remote_path, files_bytes = None):
|
||||
sha256 = self.digest(local_path) #sha value at source
|
||||
total_attempts = int(self.remote_attempts)
|
||||
local_path_str = str(local_path)
|
||||
local_path_get = ''.join([local_path_str[:local_path_str.rindex('/')]])
|
||||
local_temp_folder = local_path_get + "/tempfolder/"
|
||||
try:
|
||||
self.connect()
|
||||
#print('remote_path chdir: ', remote_path)
|
||||
self.sftp.chdir(remote_path) # Test if remote_path exists
|
||||
except IOError:
|
||||
self.sftp.mkdir(remote_path)
|
||||
finally:
|
||||
self.disconnect()
|
||||
if not os.path.isdir(local_temp_folder):
|
||||
original_umask = os.umask(0) #create local temp folder for redownloaded files
|
||||
os.makedirs(local_temp_folder, 755)
|
||||
os.umask(original_umask)
|
||||
|
||||
filename = local_path_str.split("/")[-1]
|
||||
remote_path = remote_path + filename
|
||||
local_temp_folder = local_temp_folder + filename
|
||||
while total_attempts > 0:
|
||||
self.connect()
|
||||
self.sftp.put(local_path, remote_path)
|
||||
self.disconnect()
|
||||
self.connect()
|
||||
self.sftp.get(remote_path, local_temp_folder)
|
||||
self.disconnect()
|
||||
local_sha256 = self.digest(local_temp_folder)
|
||||
os.remove(local_temp_folder)
|
||||
if local_sha256 == sha256:
|
||||
#print('sha confirmed')
|
||||
return True
|
||||
total_attempts = self.subtract_remote_attempts(total_attempts)
|
||||
if total_attempts == 0:
|
||||
return False
|
||||
|
||||
|
||||
def delete_remote(self, source_files):
|
||||
self.connect()
|
||||
for pathfile in source_files:
|
||||
pathfile = str(pathfile)
|
||||
if not pathfile.endswith('.dem'):
|
||||
continue
|
||||
filename = pathfile.split("/")[-1]
|
||||
pathfile = self.path + filename
|
||||
utime = self.sftp.stat(pathfile).st_mtime
|
||||
last_modified = datetime.fromtimestamp(utime)
|
||||
if (datetime.now() - last_modified) > timedelta(days=30 * 1.5):
|
||||
print('deleting file remotely: ', pathfile)
|
||||
self.sftp.remove(pathfile)
|
||||
self.disconnect()
|
||||
@ -1 +0,0 @@
|
||||
paramiko
|
||||
@ -1,4 +0,0 @@
|
||||
#!/bin/bash
|
||||
cd $(dirname $0)
|
||||
source venv/bin/activate
|
||||
python3 demo_mover.py config.json
|
||||
@ -1,9 +0,0 @@
|
||||
[Unit]
|
||||
Description=Moves demos from the local machine to remote destinations every 5 minutes
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=gameservers
|
||||
WorkingDirectory=/home/gameservers/demo_mover
|
||||
ExecStart=/home/gameservers/demo_mover/run_demo_mover.sh
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
[Unit]
|
||||
Description=Moves demos from the local machine to remote destinations every 5 minutes
|
||||
Requires=demo_mover.service
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* *:0,5,10,15,20,25,30,35,40,45,50,55
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Loading…
Reference in New Issue
Block a user