removed css content from cloudflare run. added new setup for OVH object storage for fastdl for css content

This commit is contained in:
jenz 2025-08-06 00:30:00 +02:00
parent 9bdaa4f383
commit 3a56e55c8a
9 changed files with 384 additions and 33 deletions

View File

@ -1,2 +1,27 @@
requires the machine that hosts the fastdl to have vsftpd running. also php-fpm and nginx.
for php-fpm remember to potentially change the www-data user to the nginx user. check in the /etc/php/ folder for it.
bucket notes:
rclone conf location:
/home/username/.config/rclone/rclone.conf
OVH bucket ACL access example:
aws s3api put-bucket-acl \
--bucket bucket-name-here \
--acl public-read \
--endpoint-url https://s3.<region>.io.cloud.ovh.net/
OVH delete versions of files from one specific bucket:
rclone purge ovh_toronto:toronto-bucket --s3-versions
OVH upload lifecycle.json to a bucket:
aws s3api put-bucket-lifecycle-configuration --bucket unloze-uk-bucket --lifecycle-configuration file://lifecycle.json --endpoint-url https://s3.uk.io.cloud.ovh.net/
OVH review that the json file is active:
aws s3api get-bucket-lifecycle-configuration --bucket unloze-uk-bucket --endpoint-url https://s3.uk.io.cloud.ovh.net/
OVH webpannel: create a replication rule from the UK machine to the other buckets, add user to the new bucket, also remember each new bucket needs versioning enabled for replication.
afterwards remember to update the gameserver config file with the virtual hosted style endpoint and longtitude and lattitude added.

View File

@ -0,0 +1,20 @@
{
"Rules": [
{
"ID": "DeleteOldVersions",
"Status": "Enabled",
"Filter": {},
"NoncurrentVersionExpiration": {
"NoncurrentDays": 1
}
},
{
"ID": "RemoveExpiredDeleteMarkers",
"Status": "Enabled",
"Filter": {},
"Expiration": {
"ExpiredObjectDeleteMarker": true
}
}
]
}

View File

@ -3,50 +3,28 @@
if [ $# -eq 1 ]
then
echo 'doing long run with all files being updated.'
#ZE server
rclone copy /home/gameservers/fastdl/css_ze/materials r2demo:/css_ze/materials/
rclone copy /home/gameservers/fastdl/css_ze/models r2demo:/css_ze/models/
rclone copy /home/gameservers/fastdl/css_ze/maps r2demo:/css_ze/maps/
rclone copy /home/gameservers/fastdl/css_ze/sound r2demo:/css_ze/sound/
#ZR server
rclone copy /home/gameservers/fastdl/css_zr/materials r2demo:/css_zr/materials/
rclone copy /home/gameservers/fastdl/css_zr/models r2demo:/css_zr/models/
rclone copy /home/gameservers/fastdl/css_zr/maps r2demo:/css_zr/maps/
rclone copy /home/gameservers/fastdl/css_zr/sound r2demo:/css_zr/sound/
#MG server
rclone copy /home/gameservers/fastdl/css_mg/materials r2demo:/css_mg/materials/
rclone copy /home/gameservers/fastdl/css_mg/models r2demo:/css_mg/models/
rclone copy /home/gameservers/fastdl/css_mg/maps r2demo:/css_mg/maps/
rclone copy /home/gameservers/fastdl/css_mg/sound r2demo:/css_mg/sound/
#svencoop
rclone copy /home/gameservers/fastdl/svencoop/svencoop_addon r2demo:/svencoop/
#gmod zs
rclone copy --no-update-modtime /home/gameservers/fastdl/gmod_zs/maps r2demo:/gmod_zs/maps/
rclone copy --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/maps r2demo:/gmod_zs/maps/
rclone copy --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/materials r2demo:/gmod_zs/materials/
rclone copy --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/models r2demo:/gmod_zs/models/
rclone copy --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/sound r2demo:/gmod_zs/sound/
rclone copy --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/resource r2demo:/gmod_zs/resource/
rclone copy --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/particles r2demo:/gmod_zs/particles/
echo 'Finished long run with all files updated.'
#else if no arguement given run default with 30minute maximum.
else
echo 'doing short run with files younger than 30 minutes.'
#ZE server
rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_ze/materials r2demo:/css_ze/materials/
rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_ze/models r2demo:/css_ze/models/
rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_ze/maps r2demo:/css_ze/maps/
rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_ze/sound r2demo:/css_ze/sound/
#ZR server
rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_zr/materials r2demo:/css_zr/materials/
rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_zr/models r2demo:/css_zr/models/
rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_zr/maps r2demo:/css_zr/maps/
rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_zr/sound r2demo:/css_zr/sound/
#MG server
rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_mg/materials r2demo:/css_mg/materials/
rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_mg/models r2demo:/css_mg/models/
rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_mg/maps r2demo:/css_mg/maps/
rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/css_mg/sound r2demo:/css_mg/sound/
#svencoop
rclone copy --max-age 30m --no-traverse /home/gameservers/fastdl/svencoop/svencoop_addon r2demo:/svencoop/
#gmod zs
rclone copy --max-age 30m --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/maps r2demo:/gmod_zs/maps/
rclone copy --max-age 30m --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/gamemodes r2demo:/gmod_zs/gamemodes/
rclone copy --max-age 30m --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/addons r2demo:/gmod_zs/addons/
rclone copy --max-age 30m --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/materials r2demo:/gmod_zs/materials/
rclone copy --max-age 30m --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/models r2demo:/gmod_zs/models/
rclone copy --max-age 30m --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/sound r2demo:/gmod_zs/sound/
rclone copy --max-age 30m --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/resource r2demo:/gmod_zs/resource/
rclone copy --max-age 30m --no-traverse --no-update-modtime /home/gameservers/fastdl/gmod_zs/particles r2demo:/gmod_zs/particles/
echo 'Finished short run with files younger than 30 minutes.'
fi

View File

@ -0,0 +1,54 @@
"FastDL"
{
"Nodes"
{
"Node 1"
{
"latitude" "43.698833328425955"
"longitude" "-79.45767829657187"
"Link" "https://unloze-toronto-bucket.s3.ca-east-tor.io.cloud.ovh.net/css_ze/"
}
"Node 2"
{
"latitude" "51.5042400586821"
"longitude" "-0.13492789756148504"
"Link" "https://unloze-uk-bucket.s3.uk.io.cloud.ovh.net/css_ze/"
}
"Node 3"
{
"latitude" "50.69328500887432"
"longitude" "3.1783407406469077"
"Link" "https://unloze-roubaix-bucket.s3.rbx.io.cloud.ovh.net/css_ze/"
}
"Node 4"
{
"latitude" "50.112167937779354"
"longitude" "8.683416865997126"
"Link" "https://unloze-ger-bucket.s3.de.io.cloud.ovh.net/css_ze/"
}
"Node 5"
{
"latitude" "52.23049810552634"
"longitude" "21.01254029979222"
"Link" "https://unloze-poland-bucket.s3.waw.io.cloud.ovh.net/css_ze/"
}
"Node 6"
{
"latitude" "18.991122638173092"
"longitude" "72.82101803450632"
"Link" "https://unloze-india-bucket.s3.ap-south-mum.io.cloud.ovh.net/css_ze/"
}
"Node 7"
{
"latitude" "1.3071592345799738"
"longitude" "103.82317573943546"
"Link" "https://unloze-singapore-bucket.s3.sgp.io.cloud.ovh.net/css_ze/"
}
"Node 8"
{
"latitude" "-33.878727424666934"
"longitude" "151.19973099759613"
"Link" "https://unloze-australia-bucket.s3.ap-southeast-syd.io.cloud.ovh.net/css_ze/"
}
}
}

View File

@ -0,0 +1,26 @@
"Games"
{
"cstrike"
{
"Signatures"
{
"CBaseClient::SendServerInfo"
{
"library" "engine"
"linux" "@_ZN11CBaseClient14SendServerInfoEv"
}
"Host_BuildConVarUpdateMessage"
{
"library" "engine"
"linux" "@_Z29Host_BuildConVarUpdateMessageP13NET_SetConVarib"
}
}
"Offsets"
{
"CBaseClient::GetPlayerSlot"
{
"linux" "3"
}
}
}
}

View File

@ -0,0 +1,184 @@
#include <sourcemod>
#include <dhooks>
#include <sdktools>
#include <geoip>
#pragma semicolon 1
#pragma newdecls required
GameData gamedatafile; //Handle to the gamedata
Handle hPlayerSlot = INVALID_HANDLE;
KeyValues nodeConfig; //main node file
char originalConVar[256]; //original sv_downloadurl value
char clientIPAddress[64]; //IP of the connecting client
int url_length = 512;
ConVar downloadurl; // sv_downloadurl
//unloze is just using geoip2, no reason to keep SypexGeo for us.
public Plugin myinfo =
{
name = "Smart Fast Downloads",
description = "Routes clients to the closest Fast Download server available. removed IP lookup feature because its not useable with S3 objects",
author = "Nolo001, edits by jenz for unloze",
url = "original is at: https://github.com/Nolo001-Aha/sourcemod_smart_fastdownloads",
version = "1.2"
};
public void OnPluginStart()
{
PrintToServer("[Smart Fast Downloads] Initializing...");
gamedatafile = LoadGameConfigFile("betterfastdl.games");
if(gamedatafile == null)
SetFailState("Cannot load betterfastdl.games.txt! Make sure you have it installed!");
Handle detourSendServerInfo = DHookCreateDetour(Address_Null, CallConv_THISCALL, ReturnType_Bool, ThisPointer_Address);
if(detourSendServerInfo == null)
SetFailState("Failed to create detour for CBaseClient::SendServerInfo!");
if(!DHookSetFromConf(detourSendServerInfo, gamedatafile, SDKConf_Signature, "CBaseClient::SendServerInfo"))
SetFailState("Failed to load CBaseClient::SendServerInfo signature from gamedata!");
if(!DHookEnableDetour(detourSendServerInfo, false, sendServerInfoDetCallback_Pre))
SetFailState("Failed to detour CBaseClient::SendServerInfo PreHook!");
Handle detourBuildConVarMessage = DHookCreateDetour(Address_Null, CallConv_CDECL, ReturnType_Void, ThisPointer_Ignore);
if(detourBuildConVarMessage == null)
SetFailState("Failed to create detour for Host_BuildConVarUpdateMessage!");
if(!DHookSetFromConf(detourBuildConVarMessage, gamedatafile, SDKConf_Signature, "Host_BuildConVarUpdateMessage"))
SetFailState("Failed to load Host_BuildConVarUpdateMessage signature from gamedata!");
DHookAddParam(detourBuildConVarMessage, HookParamType_Unknown);
DHookAddParam(detourBuildConVarMessage, HookParamType_Int);
DHookAddParam(detourBuildConVarMessage, HookParamType_Bool);
if(!DHookEnableDetour(detourBuildConVarMessage, false, buildConVarMessageDetCallback_Pre))
SetFailState("Failed to detour Host_BuildConVarUpdateMessage PreHook!");
if(!DHookEnableDetour(detourBuildConVarMessage, true, buildConVarMessageDetCallback_Post))
SetFailState("Failed to detour Host_BuildConVarUpdateMessage PostHook!");
StartPrepSDKCall(SDKCall_Raw);
PrepSDKCall_SetFromConf(gamedatafile, SDKConf_Virtual, "CBaseClient::GetPlayerSlot");
PrepSDKCall_SetReturnInfo(SDKType_PlainOldData, SDKPass_Plain);
hPlayerSlot = EndPrepSDKCall();
nodeConfig = new KeyValues("FastDL Settings"); //Load the main node config file
char config[PLATFORM_MAX_PATH];
BuildPath(Path_SM, config, sizeof(config), "configs/fastdlmanager.cfg");
nodeConfig.ImportFromFile(config);
nodeConfig.Rewind();
BuildPath(Path_SM, config, sizeof(config), "fastdl_debug.log");
AutoExecConfig(true, "SmartFastDownloads");
downloadurl = FindConVar("sv_downloadurl"); //Save original downloadurl, so we can send it to clients who we cant locate
downloadurl.GetString(originalConVar, sizeof(originalConVar));
}
public void OnConfigsExecuted()
{
downloadurl = FindConVar("sv_downloadurl");
downloadurl.GetString(originalConVar, sizeof(originalConVar));
}
public MRESReturn sendServerInfoDetCallback_Pre(Address pointer, Handle hReturn, Handle hParams) //First callback in chain, derive client and find their IP
{
int client;
client = view_as<int>(SDKCall(hPlayerSlot, pointer)) + 1; //we just use linux anyways, no reason to check OS.
GetClientIP(client, clientIPAddress, sizeof(clientIPAddress));
return MRES_Ignored;
}
public MRESReturn buildConVarMessageDetCallback_Pre(Handle hParams) //Second callback in chain, call our main function and get a node link in response
{
char[] url = new char[url_length];
getLocationSettings(url);
setConVarValue(url);
return MRES_Ignored;
}
void getLocationSettings(char[] link) //Main function
{
nodeConfig.Rewind();
float clientLongitude, clientLatitude;
if(nodeConfig.JumpToKey("Nodes", false))
{
char[] nodeURL = new char[url_length];
char[] finalurl = new char[url_length];
float distance, currentDistance; //1 - distance to the closest server, may change in iterations. 2 - distance between client and current iteration node
clientLatitude = GetLatitude(clientIPAddress); //clients coordinates
clientLongitude = GetLongitude(clientIPAddress);
//if look up of client failed just give them the default fastdl value instead.
if(clientLatitude == 0 || clientLongitude == 0)
{
//LogMessage("Failed distance calculation. Sending default values. Client(%f %f IP: %s).", clientLatitude, clientLongitude, clientIPAddress);
strcopy(link, url_length, "EMPTY");
return;
}
char section[64];
nodeConfig.GotoFirstSubKey(false);
do
{
float nodeLongitude, nodeLatitude;
nodeConfig.GetSectionName(section, sizeof(section));
nodeLatitude = nodeConfig.GetFloat("latitude");
nodeLongitude = nodeConfig.GetFloat("longitude");
currentDistance = GetDistance(nodeLatitude, nodeLongitude, clientLatitude, clientLongitude);
if((currentDistance < distance) || distance == 0)
{
nodeConfig.GetString("link", nodeURL, url_length, "EMPTY");
strcopy(finalurl, url_length, nodeURL);
distance = currentDistance;
}
}
while(nodeConfig.GotoNextKey(false));
strcopy(link, url_length, nodeURL);
//LogMessage("Sending final: Distance is %f. Client IP Address: %s. selected link is: %s", distance, clientIPAddress, link);
}
}
void setConVarValue(char[] value) //Sets the actual ConVar value
{
int oldflags = GetConVarFlags(downloadurl);
SetConVarFlags(downloadurl, oldflags &~ FCVAR_REPLICATED);
if (StrEqual(value, "EMPTY", false))
{
SetConVarString(downloadurl, originalConVar, true, false);
}
else
{
SetConVarString(downloadurl, value, true, false);
}
SetConVarFlags(downloadurl, oldflags|FCVAR_REPLICATED);
}
public MRESReturn buildConVarMessageDetCallback_Post(Handle hParams) //Reverts the ConVar to its original value
{
setConVarValue("EMPTY");
return MRES_Ignored;
}
float GetLatitude(char[] lookupIP)
{
return GeoipLatitude(lookupIP);
}
float GetLongitude(char[] lookupIP)
{
return GeoipLongitude(lookupIP);
}
float GetDistance(float nodeLatitude, float nodeLongitude, float clientLatitude, float clientLongitude)
{
return GeoipDistance(nodeLatitude, nodeLongitude, clientLatitude, clientLongitude);
}

View File

@ -0,0 +1,9 @@
[Unit]
Description=Rclones to OVH bucket every 10 minutes
[Service]
Type=simple
User=gameservers
Group=gameservers
WorkingDirectory=/home/gameservers/fastdl_sync
ExecStart=/home/gameservers/fastdl_sync/update_ovh_bucket.sh

View File

@ -0,0 +1,9 @@
[Unit]
Description=rclones new content every 10 minutes to OVH
Requires=ovh_bucket_rclone.service
[Timer]
OnCalendar=*-*-* *:0,10,20,30,40,50
[Install]
WantedBy=multi-user.target

View File

@ -0,0 +1,46 @@
#!/usr/bin/sh
#run full reclone when arguement is given
if [ $# -eq 1 ]
then
echo 'doing long run with all files being updated.'
#ZE server
rclone copy --s3-storage-class EXPRESS_ONEZONE --no-traverse /home/gameservers/fastdl/css_ze/materials ovh_uk:unloze-uk-bucket/css_ze/materials/
rclone copy --s3-storage-class EXPRESS_ONEZONE --no-traverse /home/gameservers/fastdl/css_ze/models ovh_uk:unloze-uk-bucket/css_ze/models/
rclone copy --s3-storage-class EXPRESS_ONEZONE --no-traverse /home/gameservers/fastdl/css_ze/sound ovh_uk:unloze-uk-bucket/css_ze/sound/
rclone copy --s3-storage-class EXPRESS_ONEZONE --no-traverse /home/gameservers/fastdl/css_ze/maps ovh_uk:unloze-uk-bucket/css_ze/maps/
#ZR server
rclone copy --s3-storage-class EXPRESS_ONEZONE --no-traverse /home/gameservers/fastdl/css_zr/materials ovh_uk:unloze-uk-bucket/css_zr/materials/
rclone copy --s3-storage-class EXPRESS_ONEZONE --no-traverse /home/gameservers/fastdl/css_zr/models ovh_uk:unloze-uk-bucket/css_zr/models/
rclone copy --s3-storage-class EXPRESS_ONEZONE --no-traverse /home/gameservers/fastdl/css_zr/sound ovh_uk:unloze-uk-bucket/css_zr/sound/
rclone copy --s3-storage-class EXPRESS_ONEZONE --no-traverse /home/gameservers/fastdl/css_zr/maps ovh_uk:unloze-uk-bucket/css_zr/maps/
#MG server
rclone copy --s3-storage-class EXPRESS_ONEZONE --no-traverse /home/gameservers/fastdl/css_mg/materials ovh_uk:unloze-uk-bucket/css_mg/materials/
rclone copy --s3-storage-class EXPRESS_ONEZONE --no-traverse /home/gameservers/fastdl/css_mg/models ovh_uk:unloze-uk-bucket/css_mg/models/
rclone copy --s3-storage-class EXPRESS_ONEZONE --no-traverse /home/gameservers/fastdl/css_mg/sound ovh_uk:unloze-uk-bucket/css_mg/sound/
rclone copy --s3-storage-class EXPRESS_ONEZONE --no-traverse /home/gameservers/fastdl/css_mg/maps ovh_uk:unloze-uk-bucket/css_mg/maps/
echo 'Finished long run with all files updated.'
#else if no arguement given run default with 30 minute maximum age on files.
else
echo 'doing short run with files younger than 30 minutes.'
#ZE server
rclone copy --s3-storage-class EXPRESS_ONEZONE --max-age 30m --no-traverse /home/gameservers/fastdl/css_ze/materials ovh_uk:unloze-uk-bucket/css_ze/materials/
rclone copy --s3-storage-class EXPRESS_ONEZONE --max-age 30m --no-traverse /home/gameservers/fastdl/css_ze/models ovh_uk:unloze-uk-bucket/css_ze/models/
rclone copy --s3-storage-class EXPRESS_ONEZONE --max-age 30m --no-traverse /home/gameservers/fastdl/css_ze/sound ovh_uk:unloze-uk-bucket/css_ze/sound/
rclone copy --s3-storage-class EXPRESS_ONEZONE --max-age 30m --no-traverse /home/gameservers/fastdl/css_ze/maps ovh_uk:unloze-uk-bucket/css_ze/maps/
#ZR server
rclone copy --s3-storage-class EXPRESS_ONEZONE --max-age 30m --no-traverse /home/gameservers/fastdl/css_zr/materials ovh_uk:unloze-uk-bucket/css_zr/materials/
rclone copy --s3-storage-class EXPRESS_ONEZONE --max-age 30m --no-traverse /home/gameservers/fastdl/css_zr/models ovh_uk:unloze-uk-bucket/css_zr/models/
rclone copy --s3-storage-class EXPRESS_ONEZONE --max-age 30m --no-traverse /home/gameservers/fastdl/css_zr/sound ovh_uk:unloze-uk-bucket/css_zr/sound/
rclone copy --s3-storage-class EXPRESS_ONEZONE --max-age 30m --no-traverse /home/gameservers/fastdl/css_zr/maps ovh_uk:unloze-uk-bucket/css_zr/maps/
#MG server
rclone copy --s3-storage-class EXPRESS_ONEZONE --max-age 30m --no-traverse /home/gameservers/fastdl/css_mg/materials ovh_uk:unloze-uk-bucket/css_mg/materials/
rclone copy --s3-storage-class EXPRESS_ONEZONE --max-age 30m --no-traverse /home/gameservers/fastdl/css_mg/models ovh_uk:unloze-uk-bucket/css_mg/models/
rclone copy --s3-storage-class EXPRESS_ONEZONE --max-age 30m --no-traverse /home/gameservers/fastdl/css_mg/sound ovh_uk:unloze-uk-bucket/css_mg/sound/
rclone copy --s3-storage-class EXPRESS_ONEZONE --max-age 30m --no-traverse /home/gameservers/fastdl/css_mg/maps ovh_uk:unloze-uk-bucket/css_mg/maps/
echo 'Finished short run with files younger than 30 minutes.'
fi