AI slop update to switch warp docker ip on bot block

torchlight_changes_unloze/torchlight3/Torchlight/CommandHandler.py

@@ -89,7 +89,7 @@ class CommandHandler():
                         Ret = await Command._rfunc(line, RMatch, player)
                     else:
                         #self.Logger.debug(f"_rfunc line: {line}")
-                        if line.startswith("!yt"): #transfering the line if its !yt or !yts in case people want to include pitch or tempo.
+                        if line.lower().startswith("!yt"): #transfering the line if its !yt or !yts in case people want to include pitch or tempo. (case-insensitive: trigger matching itself is case-insensitive, so this must be too)
                             Ret = await Command._func(Message, player, line)
                         else:
                             Ret = await Command._func(Message, player)

torchlight_changes_unloze/torchlight3/Torchlight/Commands.py

@@ -8,6 +8,58 @@ import math
 from .Utils import Utils, DataHolder
 import traceback
 
+# Path to the WARP IP-rotation script. Must be executable (chmod +x) and live
+# in the same folder as _start.sh (the systemd unit's WorkingDirectory), since
+# rotate_warp.sh itself cd's into the warp-docker compose folder.
+WARP_ROTATE_SCRIPT = "/home/gameservers/css_ze/torchlight3/rotate_warp.sh"
+WARP_ROTATE_TIMEOUT = 60  # seconds to allow rotate_warp.sh to bring WARP back up
+
+# yt-dlp's stderr when YouTube's bot-check trips. Matched as two substrings
+# rather than one exact string since the apostrophe yt-dlp prints can vary
+# (straight vs curly quote) between versions.
+YOUTUBE_BOT_CHECK_MARKERS = (b"Sign in to confirm", b"not a bot")
+
+
+async def run_ytdlp_with_rotation(logger, *args):
+    """
+    Run `yt-dlp *args` exactly as before. If yt-dlp's stderr shows YouTube's
+    "Sign in to confirm you're not a bot" bot-check, rotate the WARP proxy's
+    IP once (via rotate_warp.sh) and transparently retry the exact same
+    yt-dlp call before giving up. The caller (and the end user) never sees
+    the first, blocked attempt -- they just get a result, possibly a couple
+    seconds slower.
+
+    Returns (stdout_bytes, stderr_bytes) from the final attempt.
+    """
+    Out, Err = b"", b""
+    for attempt in (1, 2):
+        Proc = await asyncio.create_subprocess_exec(
+            "yt-dlp", *args,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE)
+        Out, Err = await Proc.communicate()
+
+        BotChecked = all(marker in Err for marker in YOUTUBE_BOT_CHECK_MARKERS)
+        if BotChecked and attempt == 1:
+            logger.warning("yt-dlp hit YouTube's bot-check, rotating WARP IP and retrying: %s", args)
+            try:
+                RotateProc = await asyncio.create_subprocess_exec(
+                    WARP_ROTATE_SCRIPT,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE)
+                RotateOut, RotateErr = await asyncio.wait_for(
+                    RotateProc.communicate(), timeout=WARP_ROTATE_TIMEOUT)
+                if RotateProc.returncode != 0:
+                    logger.error("rotate_warp.sh failed (rc=%s): %s", RotateProc.returncode,
+                        RotateErr.decode(errors="replace"))
+            except (asyncio.TimeoutError, OSError) as e:
+                logger.error("WARP rotation did not complete in time, retrying with same IP anyway: %s", e)
+            continue
+        break
+
+    return Out, Err
+
+
 def get_birtate(message):
     bitrate = []
     try:
@@ -114,18 +166,22 @@ class URLFilter(BaseCommand):
                 if TimeStr:
                     Time = Utils.ParseTime(TimeStr)
 
-            Proc = await asyncio.create_subprocess_exec("yt-dlp", "--dump-json", "-g", url,
-                stdout = asyncio.subprocess.PIPE)
-            Out, _ = await Proc.communicate()
+            # No "-g" here: we only need metadata (title/duration/webpage_url)
+            # for the chat message. Actual playback happens by handing the
+            # canonical webpage URL to yt-dlp again at play time (see
+            # YTDLP_URI_PREFIX in FFmpegAudioPlayer.py) rather than resolving
+            # to a direct googlevideo URL here and having a separate process
+            # (ffmpeg) try to fetch it -- that signed-URL handoff turned out
+            # to reliably 403 even with matching proxy/headers.
+            Out, Err = await run_ytdlp_with_rotation(self.Logger, "--dump-json", "--no-playlist", url)
 
-            parts = Out.split(b'\n')
-            parts.pop() # trailing new line
+            if not Out.strip():
+                self.Logger.error("yt-dlp returned no output for %s: %s", url, Err.decode(errors = "replace"))
+                self.Torchlight().SayChat("Error: Could not fetch YouTube info, try again in a moment.")
+                return url, Text
 
-            Info = parts.pop()
-            url = parts.pop()
-
-            url = url.strip().decode("ascii")
-            Info = self.json.loads(Info)
+            Info = self.json.loads(Out.strip().split(b'\n')[-1])
+            url = "ytdlp:" + Info.get("webpage_url", url)
 
             if Info["extractor_key"] == "Youtube":
                 self.Torchlight().SayChat("\x07E52D27[YouTube]\x01 {0} | {1} | {2:,}".format(
@@ -704,14 +760,19 @@ class YouTubeSearch(BaseCommand):
             message[1] = message[1][:Temp.value]
 
         search_term = message[1].split("pitch=")[0].split("tempo=")[0].split('backward=')[0].split('backwards=')[0]
-        Proc = await asyncio.create_subprocess_exec("yt-dlp", "--dump-json", "-xg", "ytsearch:" + search_term,
-            stdout = asyncio.subprocess.PIPE)
-        Out, _ = await Proc.communicate()
+        # No "-xg" here: only metadata is needed at this point. Playback goes
+        # through yt-dlp itself again (piped into ffmpeg) rather than a
+        # resolved googlevideo URL -- see the comment above YTDLP_URI_PREFIX
+        # in FFmpegAudioPlayer.py for why.
+        Out, Err = await run_ytdlp_with_rotation(self.Logger, "--dump-json", "--no-playlist", "ytsearch:" + search_term)
 
-        print('out value: ', Out)
-        url, Info = Out.split(b'\n', maxsplit = 1)
-        url = url.strip().decode("ascii")
-        Info = self.json.loads(Info)
+        if not Out.strip():
+            self.Logger.error("yt-dlp returned no output for search '%s': %s", search_term, Err.decode(errors = "replace"))
+            self.Torchlight().SayChat("Error: Could not fetch YouTube info, try again in a moment.")
+            return 1
+
+        Info = self.json.loads(Out.strip().split(b'\n')[-1])
+        url = "ytdlp:" + Info.get("webpage_url", "ytsearch:" + search_term)
 
         if Info["extractor_key"] == "Youtube":
             self.Torchlight().SayChat("\x07E52D27[YouTube]\x01 {0} | {1} | {2:,}".format(
@@ -987,4 +1048,3 @@ class Reload(BaseCommand):
         self.Logger.debug(sys._getframe().f_code.co_name + ' ' + str(message))
         self.Torchlight().Reload()
         return 0
-

torchlight_changes_unloze/torchlight3/Torchlight/FFmpegAudioPlayer.py

@@ -8,9 +8,27 @@ import time
 import socket
 import struct
 import sys
+import os
 
 SAMPLEBYTES = 2
 
+# Playing a googlevideo.com URL directly with ffmpeg turned out unreliable:
+# the signed URL (and the exact headers/User-Agent yt-dlp used to obtain it)
+# is tied to the specific request that generated it, and a *separate* ffmpeg
+# process making its own connection -- even through the same proxy, even with
+# matching headers copied from the --dump-json output -- still got HTTP 403
+# from Google. Matching proxy + headers wasn't enough because the header set
+# also drifts (yt-dlp's impersonated Chrome build number alone changed
+# between two of our test runs a few minutes apart), so hardcoding it is a
+# losing game.
+#
+# The reliable fix is to stop trying to replay yt-dlp's signed URL from a
+# second process at all: let yt-dlp itself fetch the media (same process,
+# same session, same proxy/headers it used to resolve the URL) and stream its
+# output straight into ffmpeg's stdin via a pipe. URIs that should be played
+# this way are prefixed with YTDLP_URI_PREFIX by Commands.py.
+YTDLP_URI_PREFIX = "ytdlp:"
+
 class FFmpegAudioPlayerFactory():
     VALID_CALLBACKS = ["Play", "Stop", "Update"]
 
@@ -50,6 +68,10 @@ class FFmpegAudioPlayer():
 
         self.Writer = None
         self.Process = None
+        # Only set in ytdlp-pipe mode (see PlayURI): the upstream yt-dlp
+        # process feeding ffmpeg's stdin. Needs to be killed alongside
+        # self.Process on Stop(), or it lingers as an orphaned process.
+        self.SourceProcess = None
 
         self.Callbacks = []
 
@@ -59,7 +81,25 @@ class FFmpegAudioPlayer():
 
     def PlayURI(self, uri, position, rubberband = None, dec_params = None, bitrate = None, 
             backwards = None, *args):
+        IsYtdlp = uri.startswith(YTDLP_URI_PREFIX)
+        # Commands.py may append a "#t=..." fragment to the URI purely so a
+        # downstream handler can re-parse it into a start position (see
+        # URLFilter.URLInfo) -- strip it back off here since the actual seek
+        # already happens via `position`/"-ss" below, and yt-dlp doesn't need
+        # (or want) an arbitrary fragment tacked onto the watch URL.
+        SourceUrl = uri[len(YTDLP_URI_PREFIX):].split("#")[0] if IsYtdlp else None
+
+        if IsYtdlp:
+            # Input is a pipe (yt-dlp's stdout), which isn't seekable, so a
+            # requested start position has to be an *output*-side "-ss"
+            # (placed after "-i") rather than the normal fast input-side
+            # seek. Slower (ffmpeg decodes and discards up to that point
+            # instead of jumping straight there) but correct.
+            Command = ["/usr/bin/ffmpeg", "-i", "pipe:0"]
             if position:
+                Command += ["-ss", str(datetime.timedelta(seconds = position))]
+            Command += ["-acodec", "pcm_s16le", "-ac", "1", "-ar", str(int(self.SampleRate)), "-f", "s16le", "-vn", *args]
+        elif position:
             PosStr = str(datetime.timedelta(seconds = position))
             Command = ["/usr/bin/ffmpeg", "-ss", PosStr, "-i", uri, "-acodec", "pcm_s16le", "-ac", "1", "-ar", str(int(self.SampleRate)), "-f", "s16le", "-vn", *args]
         else:
@@ -92,7 +132,7 @@ class FFmpegAudioPlayer():
             self.Master.Logger.debug(f"command: {Command}")
         Command += ["-"]
         #self.Master.Logger.debug(f"command: {Command}")
-        asyncio.ensure_future(self._stream_subprocess(Command))
+        asyncio.ensure_future(self._stream_subprocess(Command, SourceUrl))
         return True
 
     def Stop(self, force = True):
@@ -107,6 +147,14 @@ class FFmpegAudioPlayer():
             except ProcessLookupError:
                 pass
 
+        if self.SourceProcess:
+            try:
+                self.SourceProcess.terminate()
+                self.SourceProcess.kill()
+                self.SourceProcess = None
+            except ProcessLookupError:
+                pass
+
         if self.Writer:
             if force:
                 Socket = self.Writer.transport.get_extra_info("socket")
@@ -188,18 +236,99 @@ class FFmpegAudioPlayer():
 
         self.StoppedPlaying = time.time()
 
-    async def _stream_subprocess(self, cmd):
+    async def _stream_subprocess(self, cmd, source_url = None):
         if not self.Playing:
             return
 
         _, self.Writer = await asyncio.open_connection(self.Host[0], self.Host[1])
 
+        # The metadata fetch (talks to youtube.com) and the actual media
+        # fetch (talks to googlevideo.com) are separate TCP connections even
+        # within one yt-dlp run, and WARP's proxy doesn't guarantee the same
+        # apparent exit IP across separate connections -- Google signs the
+        # media URL to whichever IP it saw on the first connection, and a
+        # mismatched second connection gets 403'd. A retry opens a fresh
+        # connection through WARP and has a decent chance of landing on a
+        # matching IP, so we give it a few tries before giving up.
+        MaxAttempts = 3 if source_url else 1
+
+        for Attempt in range(1, MaxAttempts + 1):
+            if not self.Playing:
+                return
+
+            SourceProcess = None
+            SourceStdErrTask = None
+            StdinArg = None
+            ReadFd = None
+            if source_url:
+                # asyncio.subprocess.PIPE gives back a StreamReader, which is
+                # NOT a real file descriptor and can't be handed to a second
+                # subprocess as stdin -- so we make an actual OS pipe and pass
+                # the raw fds instead, exactly like a shell "|" would, letting
+                # the kernel move the bytes without going through this
+                # process.
+                ReadFd, WriteFd = os.pipe()
+                try:
+                    SourceProcess = await asyncio.create_subprocess_exec(
+                        "yt-dlp", "-f", "bestaudio/best", "-o", "-", "--no-playlist", "--quiet", source_url,
+                        stdout = WriteFd, stderr = asyncio.subprocess.PIPE)
+                finally:
+                    os.close(WriteFd) # our copy; the child inherited its own
+
+                self.SourceProcess = SourceProcess
+                SourceStdErrTask = asyncio.ensure_future(SourceProcess.stderr.read())
+                StdinArg = ReadFd
+
             Process = await asyncio.create_subprocess_exec(*cmd,
-                stdout = asyncio.subprocess.PIPE, stderr = asyncio.subprocess.DEVNULL)
+                    stdin = StdinArg,
+                    stdout = asyncio.subprocess.PIPE, stderr = asyncio.subprocess.PIPE)
             self.Process = Process
 
+            if source_url:
+                os.close(ReadFd) # our copy; the child inherited its own
+
+            # Drain stderr concurrently so it can't fill its pipe buffer and
+            # stall ffmpeg; we only care about it if playback produced
+            # nothing.
+            StdErrTask = asyncio.ensure_future(Process.stderr.read())
+
+            self.Seconds = 0.0
             await self._read_stream(Process.stdout, self.Writer)
             await Process.wait()
+            if SourceProcess:
+                await SourceProcess.wait()
+
+            if self.Seconds > 0.0:
+                return # success
+
+            StdErr = b""
+            try:
+                StdErr = await asyncio.wait_for(StdErrTask, timeout = 2)
+            except asyncio.TimeoutError:
+                pass
+
+            SourceStdErr = b""
+            if SourceStdErrTask:
+                try:
+                    SourceStdErr = await asyncio.wait_for(SourceStdErrTask, timeout = 2)
+                except asyncio.TimeoutError:
+                    pass
+
+            Is403 = source_url and b"403" in SourceStdErr
+
+            if Is403 and Attempt < MaxAttempts and self.Playing:
+                self.Master.Logger.warning("yt-dlp download got 403 (attempt %s/%s), retrying: %s",
+                    Attempt, MaxAttempts, source_url)
+                continue
+
+            self.Master.Logger.warning(
+                "no audio produced (ffmpeg rc=%s cmd=%s stderr=%s) (yt-dlp rc=%s url=%s stderr=%s)",
+                Process.returncode, cmd, StdErr.decode(errors = "replace")[-1500:],
+                SourceProcess.returncode if SourceProcess else None, source_url,
+                SourceStdErr.decode(errors = "replace")[-1500:])
+
+            if Is403:
+                self.Torchlight().SayChat("Error: YouTube blocked this stream after {0} attempts, try again.".format(MaxAttempts))
 
-        if self.Seconds == 0.0:
             self.Stop()
+            return

torchlight_changes_unloze/torchlight3/rotate_warp.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# rotate_warp.sh — force warp-docker to register fresh and get a new exit IP.
+# Called automatically by Commands.py (run_ytdlp_with_rotation) when yt-dlp
+# hits YouTube's "Sign in to confirm you're not a bot" check.
+#
+# Lives at /home/gameservers/css_ze/torchlight3/rotate_warp.sh — same folder
+# as _start.sh. Make sure it's executable: chmod +x rotate_warp.sh
+
+set -euo pipefail
+
+WARP_DIR="/home/gameservers/warp-docker"
+PROXY="http://127.0.0.1:1080"
+
+cd "$WARP_DIR"
+
+docker compose down
+rm -rf ./data/*
+docker compose up -d
+
+# Wait for the new WARP registration to actually come online.
+for i in $(seq 1 15); do
+    if curl -s -x "$PROXY" https://cloudflare.com/cdn-cgi/trace 2>/dev/null | grep -q "warp=on"; then
+        echo "WARP rotated, new IP confirmed"
+        exit 0
+    fi
+    sleep 2
+done
+
+echo "WARP did not come back up in time" >&2
+exit 1