diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py index 1874b09..d0a4d35 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py @@ -6,7 +6,8 @@ if typing.TYPE_CHECKING: from yt_dlp import YoutubeDL -from yt_dlp.networking.common import Request +from yt_dlp.networking._helper import select_proxy +from yt_dlp.networking.common import Features, Request from yt_dlp.networking.exceptions import RequestError, UnsupportedRequest try: @@ -21,8 +22,12 @@ @register_provider class BgUtilHTTPPotProviderRH(GetPOTProvider): _PROVIDER_NAME = 'BgUtilHTTPPot' - _SUPPORTED_CLIENTS = ('web', 'web_safari', 'web_embedded', 'web_music', 'web_creator', 'mweb', 'tv_embedded', 'tv') + _SUPPORTED_CLIENTS = ('web', 'web_safari', 'web_embedded', + 'web_music', 'web_creator', 'mweb', 'tv_embedded', 'tv') VERSION = __version__ + _SUPPORTED_PROXY_SCHEMES = ( + 'http', 'https', 'socks4', 'socks4a', 'socks5', 'socks5h') + _SUPPORTED_FEATURES = (Features.NO_PROXY, Features.ALL_PROXY) def _validate_get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=None, player_url=None, **kwargs): base_url = ydl.get_info_extractor('Youtube')._configuration_arg( @@ -31,9 +36,11 @@ def _validate_get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data raise UnsupportedRequest( 'One of [data_sync_id, visitor_data] must be passed') try: - response = ydl.urlopen(Request(f'{base_url}/ping', extensions={'timeout': 5.0})) + response = ydl.urlopen(Request( + f'{base_url}/ping', extensions={'timeout': 5.0}, proxies={'all': None})) except Exception as e: - raise UnsupportedRequest(f'Error reaching GET /ping (caused by {e!s})') from e + raise UnsupportedRequest( + f'Error reaching GET /ping (caused by {e!s})') from e try: response = json.load(response) except json.JSONDecodeError as e: @@ -51,6 +58,11 @@ def _validate_get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=None, player_url=None, **kwargs) -> str: self._logger.info('Generating POT via HTTP server') + if ((proxy := select_proxy('https://jnn-pa.googleapis.com', self.proxies)) + != select_proxy('https://youtube.com', self.proxies)): + self._logger.warning( + 'Proxies for https://youtube.com and https://jnn-pa.googleapis.com are different. ' + 'This is likely to cause subsequent errors.') try: response = ydl.urlopen(Request( @@ -58,8 +70,9 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= 'client': client, 'visitor_data': visitor_data, 'data_sync_id': data_sync_id, + 'proxy': proxy, }).encode(), headers={'Content-Type': 'application/json'}, - extensions={'timeout': 12.5})) + extensions={'timeout': 12.5}, proxies={'all': None})) except Exception as e: raise RequestError( f'Error reaching POST /get_pot (caused by {e!s})') from e diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py index 56fd57a..7e616a1 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py @@ -8,6 +8,8 @@ if typing.TYPE_CHECKING: from yt_dlp import YoutubeDL +from yt_dlp.networking._helper import select_proxy +from yt_dlp.networking.common import Features from yt_dlp.networking.exceptions import RequestError, UnsupportedRequest from yt_dlp.utils import Popen, classproperty @@ -23,8 +25,12 @@ @register_provider class BgUtilScriptPotProviderRH(GetPOTProvider): _PROVIDER_NAME = 'BgUtilScriptPot' - _SUPPORTED_CLIENTS = ('web', 'web_safari', 'web_embedded', 'web_music', 'web_creator', 'mweb', 'tv_embedded', 'tv') + _SUPPORTED_CLIENTS = ('web', 'web_safari', 'web_embedded', + 'web_music', 'web_creator', 'mweb', 'tv_embedded', 'tv') VERSION = __version__ + _SUPPORTED_PROXY_SCHEMES = ( + 'http', 'https', 'socks4', 'socks4a', 'socks5', 'socks5h') + _SUPPORTED_FEATURES = (Features.NO_PROXY, Features.ALL_PROXY) @classproperty(cache=True) def _default_script_path(self): @@ -51,8 +57,13 @@ def _validate_get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=None, player_url=None, **kwargs) -> str: self._logger.info( f'Generating POT via script: {self.script_path}') - command_args = ['node', self.script_path] + if proxy := select_proxy('https://jnn-pa.googleapis.com', self.proxies): + if proxy != select_proxy('https://youtube.com', self.proxies): + self._logger.warning( + 'Proxies for https://youtube.com and https://jnn-pa.googleapis.com are different. ' + 'This is likely to cause subsequent errors.') + command_args.extend(['-p', proxy]) if data_sync_id: command_args.extend(['-d', data_sync_id]) elif visitor_data: @@ -75,7 +86,8 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= msg += f'\nstderr:\n{stderr.strip()}' self._logger.debug(msg) if returncode: - raise RequestError(f'_get_pot_via_script failed with returncode {returncode}') + raise RequestError( + f'_get_pot_via_script failed with returncode {returncode}') try: # The JSON response is always the last line diff --git a/server/package.json b/server/package.json index fc6fd9c..937d79c 100644 --- a/server/package.json +++ b/server/package.json @@ -19,11 +19,14 @@ }, "dependencies": { "@commander-js/extra-typings": "commander-js/extra-typings", + "axios": "^1.7.7", "bgutils-js": "^1.1.0", "body-parser": "^1.20.2", "commander": "^12.1.0", "express": "^4.19.2", + "https-proxy-agent": "^7.0.5", "jsdom": "^25.0.0", + "socks-proxy-agent": "^8.0.4", "youtubei.js": "^10.4.0" }, "devDependencies": { diff --git a/server/src/generate_once.ts b/server/src/generate_once.ts index a90e6d0..14ab85d 100644 --- a/server/src/generate_once.ts +++ b/server/src/generate_once.ts @@ -7,14 +7,16 @@ const CACHE_PATH = path.resolve(__dirname, "..", "cache.json"); const program = new Command() .option("-v, --visitor-data ") .option("-d, --data-sync-id ") + .option("-p, --proxy ") .option("--verbose"); program.parse(); const options = program.opts(); (async () => { - const dataSyncId = options.dataSyncId; const visitorData = options.visitorData; + const dataSyncId = options.dataSyncId; + const proxy = options.proxy || ""; const verbose = options.verbose || false; let visitIdentifier: string; const cache: YoutubeSessionDataCaches = {}; @@ -57,7 +59,11 @@ const options = program.opts(); visitIdentifier = generatedVisitorData; } - const sessionData = await sessionManager.generatePoToken(visitIdentifier); + const sessionData = await sessionManager.generatePoToken( + visitIdentifier, + proxy, + ); + try { fs.writeFileSync( CACHE_PATH, diff --git a/server/src/main.ts b/server/src/main.ts index 6ba92e8..9c2c47d 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -25,7 +25,7 @@ const sessionManager = new SessionManager(options.verbose || false); httpServer.post("/get_pot", async (request, response) => { const visitorData = request.body.visitor_data as string; const dataSyncId = request.body.data_sync_id as string; - + const proxy: string = request.body.proxy; let visitIdentifier: string; // prioritize data sync id for authenticated requests, if passed @@ -51,7 +51,10 @@ httpServer.post("/get_pot", async (request, response) => { visitIdentifier = generatedVisitorData; } - const sessionData = await sessionManager.generatePoToken(visitIdentifier); + const sessionData = await sessionManager.generatePoToken( + visitIdentifier, + proxy, + ); response.send({ po_token: sessionData.poToken, visit_identifier: sessionData.visitIdentifier, diff --git a/server/src/session_manager.ts b/server/src/session_manager.ts index 01acbda..238b2b9 100644 --- a/server/src/session_manager.ts +++ b/server/src/session_manager.ts @@ -1,6 +1,10 @@ -import { BG } from "bgutils-js"; +import { BG, BgConfig, DescrambledChallenge } from "bgutils-js"; import { JSDOM } from "jsdom"; import { Innertube } from "youtubei.js"; +import { HttpsProxyAgent } from "https-proxy-agent"; +import axios from "axios"; +import { Agent } from "https"; +import { SocksProxyAgent } from "socks-proxy-agent"; interface YoutubeSessionData { poToken: string; @@ -12,17 +16,40 @@ export interface YoutubeSessionDataCaches { [visitIdentifier: string]: YoutubeSessionData; } -export class SessionManager { - shouldLog: boolean; +class Logger { + private shouldLog: boolean; + + constructor(shouldLog = true) { + this.shouldLog = shouldLog; + } + + debug(msg: string) { + if (this.shouldLog) console.debug(msg); + } + + log(msg: string) { + if (this.shouldLog) console.log(msg); + } + + warn(msg: string) { + if (this.shouldLog) console.warn(msg); + } + error(msg: string) { + if (this.shouldLog) console.error(msg); + } +} + +export class SessionManager { private youtubeSessionDataCaches: YoutubeSessionDataCaches = {}; private TOKEN_TTL_HOURS: number; + private logger: Logger; constructor( shouldLog = true, youtubeSessionDataCaches: YoutubeSessionDataCaches = {}, ) { - this.shouldLog = shouldLog; + this.logger = new Logger(shouldLog); this.setYoutubeSessionDataCaches(youtubeSessionDataCaches); this.TOKEN_TTL_HOURS = process.env.TOKEN_TTL ? parseInt(process.env.TOKEN_TTL) @@ -59,35 +86,62 @@ export class SessionManager { this.youtubeSessionDataCaches = youtubeSessionData || {}; } - log(msg: string) { - if (this.shouldLog) console.log(msg); - } - async generateVisitorData(): Promise { const innertube = await Innertube.create({ retrieve_player: false }); const visitorData = innertube.session.context.client.visitorData; if (!visitorData) { - console.error("Unable to generate visitor data via Innertube"); + this.logger.error("Unable to generate visitor data via Innertube"); return null; } return visitorData; } + getProxyDispatcher(proxy: string | undefined): Agent | undefined { + if (!proxy) return undefined; + let protocol: string; + try { + const parsedUrl = new URL(proxy); + protocol = parsedUrl.protocol.replace(":", ""); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + } catch (e) { + // assume http if no protocol was passed + protocol = "http"; + proxy = `http://${proxy}`; + } + + switch (protocol) { + case "http": + case "https": + this.logger.log(`Using HTTP/HTTPS proxy: ${proxy}`); + return new HttpsProxyAgent(proxy); + case "socks": + case "socks4": + case "socks4a": + case "socks5": + case "socks5h": + this.logger.log(`Using SOCKS proxy: ${proxy}`); + return new SocksProxyAgent(proxy); + default: + this.logger.warn(`Unsupported proxy protocol: ${proxy}`); + return undefined; + } + } // mostly copied from /~https://github.com/LuanRT/BgUtils/tree/main/examples/node async generatePoToken( visitIdentifier: string, + proxy: string = "", ): Promise { this.cleanupCaches(); const sessionData = this.youtubeSessionDataCaches[visitIdentifier]; if (sessionData) { - this.log( + this.logger.log( `POT for ${visitIdentifier} still fresh, returning cached token`, ); return sessionData; } - this.log( + this.logger.log( `POT for ${visitIdentifier} stale or not yet generated, generating...`, ); @@ -98,32 +152,77 @@ export class SessionManager { globalThis.window = dom.window as any; globalThis.document = dom.window.document; - const bgConfig = { - fetch: (url: any, options: any) => fetch(url, options), + let dispatcher: Agent | undefined; + if (proxy) { + dispatcher = this.getProxyDispatcher(proxy); + } else { + dispatcher = this.getProxyDispatcher( + process.env.HTTPS_PROXY || + process.env.HTTP_PROXY || + process.env.ALL_PROXY, + ); + } + + const bgConfig: BgConfig = { + fetch: async (url: any, options: any): Promise => { + try { + const response = await axios.post(url, options.body, { + headers: options.headers, + httpsAgent: dispatcher, + }); + + return { + ok: true, + json: async () => { + return response.data; + }, + }; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + } catch (e) { + return { + ok: false, + json: async () => { + return null; + }, + }; + } + }, globalObj: globalThis, identity: visitIdentifier, requestKey, }; - const challenge = await BG.Challenge.create(bgConfig); - + let challenge: DescrambledChallenge | undefined; + try { + challenge = await BG.Challenge.create(bgConfig); + } catch (e) { + throw new Error( + `Error while attempting to retrieve BG challenge. err = ${e}`, + ); + } if (!challenge) throw new Error("Could not get Botguard challenge"); - if (challenge.script) { const script = challenge.script.find((sc) => sc !== null); if (script) new Function(script)(); } else { - this.log("Unable to load Botguard."); + this.logger.log("Unable to load Botguard."); } - const poToken = await BG.PoToken.generate({ - program: challenge.challenge, - globalName: challenge.globalName, - bgConfig, - }); + let poToken: string | undefined; + try { + poToken = await BG.PoToken.generate({ + program: challenge.challenge, + globalName: challenge.globalName, + bgConfig, + }); + } catch (e) { + throw new Error( + `Error while trying to generate PO token. e = ${e}`, + ); + } - this.log(`po_token: ${poToken}`); - this.log(`visit_identifier: ${visitIdentifier}`); + this.logger.log(`po_token: ${poToken}`); + this.logger.log(`visit_identifier: ${visitIdentifier}`); if (!poToken) { throw new Error("po_token unexpected undefined"); diff --git a/server/src/version.ts b/server/src/version.ts index 5835689..90b7cf8 100644 --- a/server/src/version.ts +++ b/server/src/version.ts @@ -2,7 +2,7 @@ import * as fs from "fs"; import * as path from "path"; const packageJson = fs.readFileSync( - path.resolve(process.cwd(), "package.json"), + path.resolve(__dirname, "..", "package.json"), "utf8", ); export const VERSION = JSON.parse(packageJson).version; diff --git a/server/yarn.lock b/server/yarn.lock index d606d4f..a50837e 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -344,7 +344,7 @@ acorn@^8.11.0, acorn@^8.12.0, acorn@^8.4.1, acorn@^8.8.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.1.tgz#71616bdccbe25e27a54439e0046e89ca76df2248" integrity sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg== -agent-base@^7.0.2, agent-base@^7.1.0: +agent-base@^7.0.2, agent-base@^7.1.0, agent-base@^7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.1.tgz#bdbded7dfb096b751a2a087eeeb9664725b2e317" integrity sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA== @@ -393,6 +393,15 @@ asynckit@^0.4.0: resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== +axios@^1.7.7: + version "1.7.7" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.7.tgz#2f554296f9892a72ac8d8e4c5b79c14a91d0a47f" + integrity sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q== + dependencies: + follow-redirects "^1.15.6" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + balanced-match@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" @@ -886,6 +895,11 @@ flatted@^3.2.9: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.1.tgz#21db470729a6734d4997002f439cb308987f567a" integrity sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw== +follow-redirects@^1.15.6: + version "1.15.9" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" + integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== + form-data@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" @@ -1052,6 +1066,14 @@ inherits@2.0.4: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== +ip-address@^9.0.5: + version "9.0.5" + resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-9.0.5.tgz#117a960819b08780c3bd1f14ef3c1cc1d3f3ea5a" + integrity sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g== + dependencies: + jsbn "1.1.0" + sprintf-js "^1.1.3" + ipaddr.js@1.9.1: version "1.9.1" resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" @@ -1103,6 +1125,11 @@ js-yaml@^4.1.0: dependencies: argparse "^2.0.1" +jsbn@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040" + integrity sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A== + jsdom@^25.0.0: version "25.0.0" resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-25.0.0.tgz#d1612b4ddab85af56821b2f731e15faae135f4e1" @@ -1368,6 +1395,11 @@ proxy-addr@~2.0.7: forwarded "0.2.0" ipaddr.js "1.9.1" +proxy-from-env@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" @@ -1532,6 +1564,33 @@ side-channel@^1.0.4: get-intrinsic "^1.2.4" object-inspect "^1.13.1" +smart-buffer@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae" + integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== + +socks-proxy-agent@^8.0.4: + version "8.0.4" + resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz#9071dca17af95f483300316f4b063578fa0db08c" + integrity sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw== + dependencies: + agent-base "^7.1.1" + debug "^4.3.4" + socks "^2.8.3" + +socks@^2.8.3: + version "2.8.3" + resolved "https://registry.yarnpkg.com/socks/-/socks-2.8.3.tgz#1ebd0f09c52ba95a09750afe3f3f9f724a800cb5" + integrity sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw== + dependencies: + ip-address "^9.0.5" + smart-buffer "^4.2.0" + +sprintf-js@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.3.tgz#4914b903a2f8b685d17fdf78a70e917e872e444a" + integrity sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA== + statuses@2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63"