diff --git a/torchlight_changes_unloze/torchlight3/Torchlight/CommandHandler.py b/torchlight_changes_unloze/torchlight3/Torchlight/CommandHandler.py index 5171b87..3bbab4d 100755 --- a/torchlight_changes_unloze/torchlight3/Torchlight/CommandHandler.py +++ b/torchlight_changes_unloze/torchlight3/Torchlight/CommandHandler.py @@ -89,7 +89,7 @@ class CommandHandler(): Ret = await Command._rfunc(line, RMatch, player) else: #self.Logger.debug(f"_rfunc line: {line}") - if line.startswith("!yt"): #transfering the line if its !yt or !yts in case people want to include pitch or tempo. + if line.lower().startswith("!yt"): #transfering the line if its !yt or !yts in case people want to include pitch or tempo. (case-insensitive: trigger matching itself is case-insensitive, so this must be too) Ret = await Command._func(Message, player, line) else: Ret = await Command._func(Message, player) diff --git a/torchlight_changes_unloze/torchlight3/Torchlight/Commands.py b/torchlight_changes_unloze/torchlight3/Torchlight/Commands.py index d76dd16..5531f46 100755 --- a/torchlight_changes_unloze/torchlight3/Torchlight/Commands.py +++ b/torchlight_changes_unloze/torchlight3/Torchlight/Commands.py @@ -8,6 +8,58 @@ import math from .Utils import Utils, DataHolder import traceback +# Path to the WARP IP-rotation script. Must be executable (chmod +x) and live +# in the same folder as _start.sh (the systemd unit's WorkingDirectory), since +# rotate_warp.sh itself cd's into the warp-docker compose folder. +WARP_ROTATE_SCRIPT = "/home/gameservers/css_ze/torchlight3/rotate_warp.sh" +WARP_ROTATE_TIMEOUT = 60 # seconds to allow rotate_warp.sh to bring WARP back up + +# yt-dlp's stderr when YouTube's bot-check trips. Matched as two substrings +# rather than one exact string since the apostrophe yt-dlp prints can vary +# (straight vs curly quote) between versions. +YOUTUBE_BOT_CHECK_MARKERS = (b"Sign in to confirm", b"not a bot") + + +async def run_ytdlp_with_rotation(logger, *args): + """ + Run `yt-dlp *args` exactly as before. If yt-dlp's stderr shows YouTube's + "Sign in to confirm you're not a bot" bot-check, rotate the WARP proxy's + IP once (via rotate_warp.sh) and transparently retry the exact same + yt-dlp call before giving up. The caller (and the end user) never sees + the first, blocked attempt -- they just get a result, possibly a couple + seconds slower. + + Returns (stdout_bytes, stderr_bytes) from the final attempt. + """ + Out, Err = b"", b"" + for attempt in (1, 2): + Proc = await asyncio.create_subprocess_exec( + "yt-dlp", *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE) + Out, Err = await Proc.communicate() + + BotChecked = all(marker in Err for marker in YOUTUBE_BOT_CHECK_MARKERS) + if BotChecked and attempt == 1: + logger.warning("yt-dlp hit YouTube's bot-check, rotating WARP IP and retrying: %s", args) + try: + RotateProc = await asyncio.create_subprocess_exec( + WARP_ROTATE_SCRIPT, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE) + RotateOut, RotateErr = await asyncio.wait_for( + RotateProc.communicate(), timeout=WARP_ROTATE_TIMEOUT) + if RotateProc.returncode != 0: + logger.error("rotate_warp.sh failed (rc=%s): %s", RotateProc.returncode, + RotateErr.decode(errors="replace")) + except (asyncio.TimeoutError, OSError) as e: + logger.error("WARP rotation did not complete in time, retrying with same IP anyway: %s", e) + continue + break + + return Out, Err + + def get_birtate(message): bitrate = [] try: @@ -114,18 +166,22 @@ class URLFilter(BaseCommand): if TimeStr: Time = Utils.ParseTime(TimeStr) - Proc = await asyncio.create_subprocess_exec("yt-dlp", "--dump-json", "-g", url, - stdout = asyncio.subprocess.PIPE) - Out, _ = await Proc.communicate() + # No "-g" here: we only need metadata (title/duration/webpage_url) + # for the chat message. Actual playback happens by handing the + # canonical webpage URL to yt-dlp again at play time (see + # YTDLP_URI_PREFIX in FFmpegAudioPlayer.py) rather than resolving + # to a direct googlevideo URL here and having a separate process + # (ffmpeg) try to fetch it -- that signed-URL handoff turned out + # to reliably 403 even with matching proxy/headers. + Out, Err = await run_ytdlp_with_rotation(self.Logger, "--dump-json", "--no-playlist", url) - parts = Out.split(b'\n') - parts.pop() # trailing new line + if not Out.strip(): + self.Logger.error("yt-dlp returned no output for %s: %s", url, Err.decode(errors = "replace")) + self.Torchlight().SayChat("Error: Could not fetch YouTube info, try again in a moment.") + return url, Text - Info = parts.pop() - url = parts.pop() - - url = url.strip().decode("ascii") - Info = self.json.loads(Info) + Info = self.json.loads(Out.strip().split(b'\n')[-1]) + url = "ytdlp:" + Info.get("webpage_url", url) if Info["extractor_key"] == "Youtube": self.Torchlight().SayChat("\x07E52D27[YouTube]\x01 {0} | {1} | {2:,}".format( @@ -704,14 +760,19 @@ class YouTubeSearch(BaseCommand): message[1] = message[1][:Temp.value] search_term = message[1].split("pitch=")[0].split("tempo=")[0].split('backward=')[0].split('backwards=')[0] - Proc = await asyncio.create_subprocess_exec("yt-dlp", "--dump-json", "-xg", "ytsearch:" + search_term, - stdout = asyncio.subprocess.PIPE) - Out, _ = await Proc.communicate() + # No "-xg" here: only metadata is needed at this point. Playback goes + # through yt-dlp itself again (piped into ffmpeg) rather than a + # resolved googlevideo URL -- see the comment above YTDLP_URI_PREFIX + # in FFmpegAudioPlayer.py for why. + Out, Err = await run_ytdlp_with_rotation(self.Logger, "--dump-json", "--no-playlist", "ytsearch:" + search_term) - print('out value: ', Out) - url, Info = Out.split(b'\n', maxsplit = 1) - url = url.strip().decode("ascii") - Info = self.json.loads(Info) + if not Out.strip(): + self.Logger.error("yt-dlp returned no output for search '%s': %s", search_term, Err.decode(errors = "replace")) + self.Torchlight().SayChat("Error: Could not fetch YouTube info, try again in a moment.") + return 1 + + Info = self.json.loads(Out.strip().split(b'\n')[-1]) + url = "ytdlp:" + Info.get("webpage_url", "ytsearch:" + search_term) if Info["extractor_key"] == "Youtube": self.Torchlight().SayChat("\x07E52D27[YouTube]\x01 {0} | {1} | {2:,}".format( @@ -987,4 +1048,3 @@ class Reload(BaseCommand): self.Logger.debug(sys._getframe().f_code.co_name + ' ' + str(message)) self.Torchlight().Reload() return 0 - diff --git a/torchlight_changes_unloze/torchlight3/Torchlight/FFmpegAudioPlayer.py b/torchlight_changes_unloze/torchlight3/Torchlight/FFmpegAudioPlayer.py index 2b8db62..7a091ee 100755 --- a/torchlight_changes_unloze/torchlight3/Torchlight/FFmpegAudioPlayer.py +++ b/torchlight_changes_unloze/torchlight3/Torchlight/FFmpegAudioPlayer.py @@ -8,9 +8,27 @@ import time import socket import struct import sys +import os SAMPLEBYTES = 2 +# Playing a googlevideo.com URL directly with ffmpeg turned out unreliable: +# the signed URL (and the exact headers/User-Agent yt-dlp used to obtain it) +# is tied to the specific request that generated it, and a *separate* ffmpeg +# process making its own connection -- even through the same proxy, even with +# matching headers copied from the --dump-json output -- still got HTTP 403 +# from Google. Matching proxy + headers wasn't enough because the header set +# also drifts (yt-dlp's impersonated Chrome build number alone changed +# between two of our test runs a few minutes apart), so hardcoding it is a +# losing game. +# +# The reliable fix is to stop trying to replay yt-dlp's signed URL from a +# second process at all: let yt-dlp itself fetch the media (same process, +# same session, same proxy/headers it used to resolve the URL) and stream its +# output straight into ffmpeg's stdin via a pipe. URIs that should be played +# this way are prefixed with YTDLP_URI_PREFIX by Commands.py. +YTDLP_URI_PREFIX = "ytdlp:" + class FFmpegAudioPlayerFactory(): VALID_CALLBACKS = ["Play", "Stop", "Update"] @@ -50,6 +68,10 @@ class FFmpegAudioPlayer(): self.Writer = None self.Process = None + # Only set in ytdlp-pipe mode (see PlayURI): the upstream yt-dlp + # process feeding ffmpeg's stdin. Needs to be killed alongside + # self.Process on Stop(), or it lingers as an orphaned process. + self.SourceProcess = None self.Callbacks = [] @@ -59,7 +81,25 @@ class FFmpegAudioPlayer(): def PlayURI(self, uri, position, rubberband = None, dec_params = None, bitrate = None, backwards = None, *args): - if position: + IsYtdlp = uri.startswith(YTDLP_URI_PREFIX) + # Commands.py may append a "#t=..." fragment to the URI purely so a + # downstream handler can re-parse it into a start position (see + # URLFilter.URLInfo) -- strip it back off here since the actual seek + # already happens via `position`/"-ss" below, and yt-dlp doesn't need + # (or want) an arbitrary fragment tacked onto the watch URL. + SourceUrl = uri[len(YTDLP_URI_PREFIX):].split("#")[0] if IsYtdlp else None + + if IsYtdlp: + # Input is a pipe (yt-dlp's stdout), which isn't seekable, so a + # requested start position has to be an *output*-side "-ss" + # (placed after "-i") rather than the normal fast input-side + # seek. Slower (ffmpeg decodes and discards up to that point + # instead of jumping straight there) but correct. + Command = ["/usr/bin/ffmpeg", "-i", "pipe:0"] + if position: + Command += ["-ss", str(datetime.timedelta(seconds = position))] + Command += ["-acodec", "pcm_s16le", "-ac", "1", "-ar", str(int(self.SampleRate)), "-f", "s16le", "-vn", *args] + elif position: PosStr = str(datetime.timedelta(seconds = position)) Command = ["/usr/bin/ffmpeg", "-ss", PosStr, "-i", uri, "-acodec", "pcm_s16le", "-ac", "1", "-ar", str(int(self.SampleRate)), "-f", "s16le", "-vn", *args] else: @@ -92,7 +132,7 @@ class FFmpegAudioPlayer(): self.Master.Logger.debug(f"command: {Command}") Command += ["-"] #self.Master.Logger.debug(f"command: {Command}") - asyncio.ensure_future(self._stream_subprocess(Command)) + asyncio.ensure_future(self._stream_subprocess(Command, SourceUrl)) return True def Stop(self, force = True): @@ -107,6 +147,14 @@ class FFmpegAudioPlayer(): except ProcessLookupError: pass + if self.SourceProcess: + try: + self.SourceProcess.terminate() + self.SourceProcess.kill() + self.SourceProcess = None + except ProcessLookupError: + pass + if self.Writer: if force: Socket = self.Writer.transport.get_extra_info("socket") @@ -188,18 +236,99 @@ class FFmpegAudioPlayer(): self.StoppedPlaying = time.time() - async def _stream_subprocess(self, cmd): + async def _stream_subprocess(self, cmd, source_url = None): if not self.Playing: return _, self.Writer = await asyncio.open_connection(self.Host[0], self.Host[1]) - Process = await asyncio.create_subprocess_exec(*cmd, - stdout = asyncio.subprocess.PIPE, stderr = asyncio.subprocess.DEVNULL) - self.Process = Process + # The metadata fetch (talks to youtube.com) and the actual media + # fetch (talks to googlevideo.com) are separate TCP connections even + # within one yt-dlp run, and WARP's proxy doesn't guarantee the same + # apparent exit IP across separate connections -- Google signs the + # media URL to whichever IP it saw on the first connection, and a + # mismatched second connection gets 403'd. A retry opens a fresh + # connection through WARP and has a decent chance of landing on a + # matching IP, so we give it a few tries before giving up. + MaxAttempts = 3 if source_url else 1 - await self._read_stream(Process.stdout, self.Writer) - await Process.wait() + for Attempt in range(1, MaxAttempts + 1): + if not self.Playing: + return + + SourceProcess = None + SourceStdErrTask = None + StdinArg = None + ReadFd = None + if source_url: + # asyncio.subprocess.PIPE gives back a StreamReader, which is + # NOT a real file descriptor and can't be handed to a second + # subprocess as stdin -- so we make an actual OS pipe and pass + # the raw fds instead, exactly like a shell "|" would, letting + # the kernel move the bytes without going through this + # process. + ReadFd, WriteFd = os.pipe() + try: + SourceProcess = await asyncio.create_subprocess_exec( + "yt-dlp", "-f", "bestaudio/best", "-o", "-", "--no-playlist", "--quiet", source_url, + stdout = WriteFd, stderr = asyncio.subprocess.PIPE) + finally: + os.close(WriteFd) # our copy; the child inherited its own + + self.SourceProcess = SourceProcess + SourceStdErrTask = asyncio.ensure_future(SourceProcess.stderr.read()) + StdinArg = ReadFd + + Process = await asyncio.create_subprocess_exec(*cmd, + stdin = StdinArg, + stdout = asyncio.subprocess.PIPE, stderr = asyncio.subprocess.PIPE) + self.Process = Process + + if source_url: + os.close(ReadFd) # our copy; the child inherited its own + + # Drain stderr concurrently so it can't fill its pipe buffer and + # stall ffmpeg; we only care about it if playback produced + # nothing. + StdErrTask = asyncio.ensure_future(Process.stderr.read()) + + self.Seconds = 0.0 + await self._read_stream(Process.stdout, self.Writer) + await Process.wait() + if SourceProcess: + await SourceProcess.wait() + + if self.Seconds > 0.0: + return # success + + StdErr = b"" + try: + StdErr = await asyncio.wait_for(StdErrTask, timeout = 2) + except asyncio.TimeoutError: + pass + + SourceStdErr = b"" + if SourceStdErrTask: + try: + SourceStdErr = await asyncio.wait_for(SourceStdErrTask, timeout = 2) + except asyncio.TimeoutError: + pass + + Is403 = source_url and b"403" in SourceStdErr + + if Is403 and Attempt < MaxAttempts and self.Playing: + self.Master.Logger.warning("yt-dlp download got 403 (attempt %s/%s), retrying: %s", + Attempt, MaxAttempts, source_url) + continue + + self.Master.Logger.warning( + "no audio produced (ffmpeg rc=%s cmd=%s stderr=%s) (yt-dlp rc=%s url=%s stderr=%s)", + Process.returncode, cmd, StdErr.decode(errors = "replace")[-1500:], + SourceProcess.returncode if SourceProcess else None, source_url, + SourceStdErr.decode(errors = "replace")[-1500:]) + + if Is403: + self.Torchlight().SayChat("Error: YouTube blocked this stream after {0} attempts, try again.".format(MaxAttempts)) - if self.Seconds == 0.0: self.Stop() + return diff --git a/torchlight_changes_unloze/torchlight3/rotate_warp.sh b/torchlight_changes_unloze/torchlight3/rotate_warp.sh new file mode 100755 index 0000000..d49f838 --- /dev/null +++ b/torchlight_changes_unloze/torchlight3/rotate_warp.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# rotate_warp.sh — force warp-docker to register fresh and get a new exit IP. +# Called automatically by Commands.py (run_ytdlp_with_rotation) when yt-dlp +# hits YouTube's "Sign in to confirm you're not a bot" check. +# +# Lives at /home/gameservers/css_ze/torchlight3/rotate_warp.sh — same folder +# as _start.sh. Make sure it's executable: chmod +x rotate_warp.sh + +set -euo pipefail + +WARP_DIR="/home/gameservers/warp-docker" +PROXY="http://127.0.0.1:1080" + +cd "$WARP_DIR" + +docker compose down +rm -rf ./data/* +docker compose up -d + +# Wait for the new WARP registration to actually come online. +for i in $(seq 1 15); do + if curl -s -x "$PROXY" https://cloudflare.com/cdn-cgi/trace 2>/dev/null | grep -q "warp=on"; then + echo "WARP rotated, new IP confirmed" + exit 0 + fi + sleep 2 +done + +echo "WARP did not come back up in time" >&2 +exit 1