initial commit of database structure, data moving and current sourcemod plugin version

RaceTimer_2026/python3/main.py

@@ -0,0 +1,120 @@
+#!/home/nonroot/unloze_racetimer_2026_insert/venv/bin/python3
+from settings import get_connection_unloze_racetimer
+from columns import MIGRATED_COLUMNS
+
+BATCH_SIZE = 2000
+FIXED_CVARS_HASH = -706992435  # FNV-1a of "CLASSIC RACETIMER ze1", same value used for the zone_categories migration
+
+
+def load_zone_category_lookup(conn):
+    """Returns {(map_name, stage): zone_category_id} for the migrated legacy categories."""
+    cur = conn.cursor()
+    cur.execute(
+        "SELECT id, map_name, stage FROM unloze_racetimer_css_2026.zone_categories WHERE cvars_hash = %s",
+        (FIXED_CVARS_HASH,),
+    )
+    lookup = {(map_name, stage): zid for zid, map_name, stage in cur.fetchall()}
+    cur.close()
+    print(f"loaded {len(lookup)} zone_category rows for lookup")
+    return lookup
+
+
+def flush_batch(write_conn, zone_lookup, batch, stats):
+    """batch: list of (steam_auth, steam_name, map_name, stage, time_value)"""
+    if not batch:
+        return
+
+    cur = write_conn.cursor()
+
+    # --- timer_improvements: one multi-row INSERT for the whole batch ---
+    placeholders = ", ".join(["(%s, %s)"] * len(batch))
+    params = []
+    for steam_auth, steam_name, _map_name, _stage, _time_value in batch:
+        params.extend((steam_auth, steam_name))
+
+    cur.execute(
+        f"INSERT INTO unloze_racetimer_css_2026.timer_improvements (steam_auth, steam_name) VALUES {placeholders}",
+        params,
+    )
+    first_id = cur.lastrowid
+    if not first_id:
+        raise RuntimeError("lastrowid was 0/None after batch insert - aborting, correlation would be wrong")
+
+    # --- timer_records: same batch, ids computed as first_id + offset ---
+    records_placeholders = ", ".join(["(%s, %s, %s)"] * len(batch))
+    records_params = []
+    skipped_no_category = 0
+    included = 0
+    for i, (steam_auth, steam_name, map_name, stage, time_value) in enumerate(batch):
+        improvement_id = first_id + i
+        zone_category_id = zone_lookup.get((map_name, stage))
+        if zone_category_id is None:
+            skipped_no_category += 1
+            continue
+        records_params.extend((improvement_id, zone_category_id, time_value))
+        included += 1
+
+    if included:
+        # rebuild placeholders in case some rows were skipped for missing categories
+        records_placeholders = ", ".join(["(%s, %s, %s)"] * included)
+        cur.execute(
+            f"INSERT INTO unloze_racetimer_css_2026.timer_records (improvement_id, zone_category_id, time_value) VALUES {records_placeholders}",
+            records_params,
+        )
+
+    write_conn.commit()
+    cur.close()
+
+    stats["improvements"] += len(batch)
+    stats["records"] += included
+    stats["skipped_no_category"] += skipped_no_category
+
+
+def main():
+    read_conn = get_connection_unloze_racetimer()
+    write_conn = get_connection_unloze_racetimer()
+    write_conn.autocommit = False
+
+    zone_lookup = load_zone_category_lookup(write_conn)
+
+    column_names = ", ".join(f"`{colname}`" for _m, _s, colname in MIGRATED_COLUMNS)
+    query = f"SELECT steam_auth, name, {column_names} FROM unloze_racetimer_css.zetimer_table_new"
+
+    # unbuffered cursor: streams rows instead of loading all ~137k into memory at once
+    read_cur = read_conn.cursor(buffered=False)
+    read_cur.execute(query)
+
+    stats = {"players": 0, "improvements": 0, "records": 0, "skipped_no_category": 0}
+    batch = []
+
+    for row in read_cur:
+        stats["players"] += 1
+        steam_auth = row[0]
+        steam_name = row[1]
+        values = row[2:]
+
+        for (map_name, stage, _colname), time_value in zip(MIGRATED_COLUMNS, values):
+            if time_value is None or time_value <= 0:
+                continue
+            batch.append((steam_auth, steam_name, map_name, stage, time_value))
+            if len(batch) >= BATCH_SIZE:
+                flush_batch(write_conn, zone_lookup, batch, stats)
+                batch = []
+
+        if stats["players"] % 5000 == 0:
+            print(f"...processed {stats['players']} players, {stats['improvements']} improvements so far")
+
+    flush_batch(write_conn, zone_lookup, batch, stats)
+
+    read_cur.close()
+    read_conn.close()
+    write_conn.close()
+
+    print("done.")
+    print(stats)
+    if stats["skipped_no_category"]:
+        print(f"WARNING: {stats['skipped_no_category']} rows had no matching zone_categories entry and were skipped - investigate before trusting the migration is complete.")
+
+
+if __name__ == '__main__':
+    main()