121 lines
4.3 KiB
Python
Executable File
121 lines
4.3 KiB
Python
Executable File
#!/home/nonroot/unloze_racetimer_2026_insert/venv/bin/python3
|
|
from settings import get_connection_unloze_racetimer
|
|
from columns import MIGRATED_COLUMNS
|
|
|
|
BATCH_SIZE = 2000
|
|
FIXED_CVARS_HASH = -706992435 # FNV-1a of "CLASSIC RACETIMER ze1", same value used for the zone_categories migration
|
|
|
|
|
|
def load_zone_category_lookup(conn):
|
|
"""Returns {(map_name, stage): zone_category_id} for the migrated legacy categories."""
|
|
cur = conn.cursor()
|
|
cur.execute(
|
|
"SELECT id, map_name, stage FROM unloze_racetimer_css_2026.zone_categories WHERE cvars_hash = %s",
|
|
(FIXED_CVARS_HASH,),
|
|
)
|
|
lookup = {(map_name, stage): zid for zid, map_name, stage in cur.fetchall()}
|
|
cur.close()
|
|
print(f"loaded {len(lookup)} zone_category rows for lookup")
|
|
return lookup
|
|
|
|
|
|
def flush_batch(write_conn, zone_lookup, batch, stats):
|
|
"""batch: list of (steam_auth, steam_name, map_name, stage, time_value)"""
|
|
if not batch:
|
|
return
|
|
|
|
cur = write_conn.cursor()
|
|
|
|
# --- timer_improvements: one multi-row INSERT for the whole batch ---
|
|
placeholders = ", ".join(["(%s, %s)"] * len(batch))
|
|
params = []
|
|
for steam_auth, steam_name, _map_name, _stage, _time_value in batch:
|
|
params.extend((steam_auth, steam_name))
|
|
|
|
cur.execute(
|
|
f"INSERT INTO unloze_racetimer_css_2026.timer_improvements (steam_auth, steam_name) VALUES {placeholders}",
|
|
params,
|
|
)
|
|
first_id = cur.lastrowid
|
|
if not first_id:
|
|
raise RuntimeError("lastrowid was 0/None after batch insert - aborting, correlation would be wrong")
|
|
|
|
# --- timer_records: same batch, ids computed as first_id + offset ---
|
|
records_placeholders = ", ".join(["(%s, %s, %s)"] * len(batch))
|
|
records_params = []
|
|
skipped_no_category = 0
|
|
included = 0
|
|
for i, (steam_auth, steam_name, map_name, stage, time_value) in enumerate(batch):
|
|
improvement_id = first_id + i
|
|
zone_category_id = zone_lookup.get((map_name, stage))
|
|
if zone_category_id is None:
|
|
skipped_no_category += 1
|
|
continue
|
|
records_params.extend((improvement_id, zone_category_id, time_value))
|
|
included += 1
|
|
|
|
if included:
|
|
# rebuild placeholders in case some rows were skipped for missing categories
|
|
records_placeholders = ", ".join(["(%s, %s, %s)"] * included)
|
|
cur.execute(
|
|
f"INSERT INTO unloze_racetimer_css_2026.timer_records (improvement_id, zone_category_id, time_value) VALUES {records_placeholders}",
|
|
records_params,
|
|
)
|
|
|
|
write_conn.commit()
|
|
cur.close()
|
|
|
|
stats["improvements"] += len(batch)
|
|
stats["records"] += included
|
|
stats["skipped_no_category"] += skipped_no_category
|
|
|
|
|
|
def main():
|
|
read_conn = get_connection_unloze_racetimer()
|
|
write_conn = get_connection_unloze_racetimer()
|
|
write_conn.autocommit = False
|
|
|
|
zone_lookup = load_zone_category_lookup(write_conn)
|
|
|
|
column_names = ", ".join(f"`{colname}`" for _m, _s, colname in MIGRATED_COLUMNS)
|
|
query = f"SELECT steam_auth, name, {column_names} FROM unloze_racetimer_css.zetimer_table_new"
|
|
|
|
# unbuffered cursor: streams rows instead of loading all ~137k into memory at once
|
|
read_cur = read_conn.cursor(buffered=False)
|
|
read_cur.execute(query)
|
|
|
|
stats = {"players": 0, "improvements": 0, "records": 0, "skipped_no_category": 0}
|
|
batch = []
|
|
|
|
for row in read_cur:
|
|
stats["players"] += 1
|
|
steam_auth = row[0]
|
|
steam_name = row[1]
|
|
values = row[2:]
|
|
|
|
for (map_name, stage, _colname), time_value in zip(MIGRATED_COLUMNS, values):
|
|
if time_value is None or time_value <= 0:
|
|
continue
|
|
batch.append((steam_auth, steam_name, map_name, stage, time_value))
|
|
if len(batch) >= BATCH_SIZE:
|
|
flush_batch(write_conn, zone_lookup, batch, stats)
|
|
batch = []
|
|
|
|
if stats["players"] % 5000 == 0:
|
|
print(f"...processed {stats['players']} players, {stats['improvements']} improvements so far")
|
|
|
|
flush_batch(write_conn, zone_lookup, batch, stats)
|
|
|
|
read_cur.close()
|
|
read_conn.close()
|
|
write_conn.close()
|
|
|
|
print("done.")
|
|
print(stats)
|
|
if stats["skipped_no_category"]:
|
|
print(f"WARNING: {stats['skipped_no_category']} rows had no matching zone_categories entry and were skipped - investigate before trusting the migration is complete.")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|