restructured manifest and plugins loading; updated plugins

plugins/youtube_download/yt_dlp/networking/_curlcffi.py

@@ -6,7 +6,8 @@ import math
 import re
 import urllib.parse
 
-from ._helper import InstanceStoreMixin, select_proxy
+from ._helper import InstanceStoreMixin
+from ..utils.networking import select_proxy
 from .common import (
     Features,
     Request,
@@ -32,9 +33,9 @@ if curl_cffi is None:
 
 curl_cffi_version = tuple(map(int, re.split(r'[^\d]+', curl_cffi.__version__)[:3]))
 
-if curl_cffi_version != (0, 5, 10) and not (0, 10) <= curl_cffi_version:
+if curl_cffi_version != (0, 5, 10) and not (0, 10) <= curl_cffi_version < (0, 14):
     curl_cffi._yt_dlp__version = f'{curl_cffi.__version__} (unsupported)'
-    raise ImportError('Only curl_cffi versions 0.5.10 and 0.10.x are supported')
+    raise ImportError('Only curl_cffi versions 0.5.10, 0.10.x, 0.11.x, 0.12.x, 0.13.x are supported')
 
 import curl_cffi.requests
 from curl_cffi.const import CurlECode, CurlOpt
@@ -119,8 +120,8 @@ BROWSER_TARGETS: dict[tuple[int, ...], dict[str, ImpersonateTarget]] = {
         'chrome110': ImpersonateTarget('chrome', '110', 'windows', '10'),
         'edge99': ImpersonateTarget('edge', '99', 'windows', '10'),
         'edge101': ImpersonateTarget('edge', '101', 'windows', '10'),
-        'safari15_3': ImpersonateTarget('safari', '15.3', 'macos', '11'),
-        'safari15_5': ImpersonateTarget('safari', '15.5', 'macos', '12'),
+        'safari153': ImpersonateTarget('safari', '15.3', 'macos', '11'),
+        'safari155': ImpersonateTarget('safari', '15.5', 'macos', '12'),
     },
     (0, 7): {
         'chrome116': ImpersonateTarget('chrome', '116', 'windows', '10'),
@@ -128,12 +129,12 @@ BROWSER_TARGETS: dict[tuple[int, ...], dict[str, ImpersonateTarget]] = {
         'chrome120': ImpersonateTarget('chrome', '120', 'macos', '14'),
         'chrome123': ImpersonateTarget('chrome', '123', 'macos', '14'),
         'chrome124': ImpersonateTarget('chrome', '124', 'macos', '14'),
-        'safari17_0': ImpersonateTarget('safari', '17.0', 'macos', '14'),
-        'safari17_2_ios': ImpersonateTarget('safari', '17.2', 'ios', '17.2'),
+        'safari170': ImpersonateTarget('safari', '17.0', 'macos', '14'),
+        'safari172_ios': ImpersonateTarget('safari', '17.2', 'ios', '17.2'),
     },
     (0, 9): {
-        'safari15_3': ImpersonateTarget('safari', '15.3', 'macos', '14'),
-        'safari15_5': ImpersonateTarget('safari', '15.5', 'macos', '14'),
+        'safari153': ImpersonateTarget('safari', '15.3', 'macos', '14'),
+        'safari155': ImpersonateTarget('safari', '15.5', 'macos', '14'),
         'chrome119': ImpersonateTarget('chrome', '119', 'macos', '14'),
         'chrome120': ImpersonateTarget('chrome', '120', 'macos', '14'),
         'chrome123': ImpersonateTarget('chrome', '123', 'macos', '14'),
@@ -142,12 +143,33 @@ BROWSER_TARGETS: dict[tuple[int, ...], dict[str, ImpersonateTarget]] = {
         'chrome131_android': ImpersonateTarget('chrome', '131', 'android', '14'),
         'chrome133a': ImpersonateTarget('chrome', '133', 'macos', '15'),
         'firefox133': ImpersonateTarget('firefox', '133', 'macos', '14'),
-        'safari18_0': ImpersonateTarget('safari', '18.0', 'macos', '15'),
-        'safari18_0_ios': ImpersonateTarget('safari', '18.0', 'ios', '18.0'),
+        'safari180': ImpersonateTarget('safari', '18.0', 'macos', '15'),
+        'safari180_ios': ImpersonateTarget('safari', '18.0', 'ios', '18.0'),
     },
     (0, 10): {
         'firefox135': ImpersonateTarget('firefox', '135', 'macos', '14'),
     },
+    (0, 11): {
+        'tor145': ImpersonateTarget('tor', '14.5', 'macos', '14'),
+        'safari184': ImpersonateTarget('safari', '18.4', 'macos', '15'),
+        'safari184_ios': ImpersonateTarget('safari', '18.4', 'ios', '18.4'),
+        'chrome136': ImpersonateTarget('chrome', '136', 'macos', '15'),
+    },
+    (0, 12): {
+        'safari260': ImpersonateTarget('safari', '26.0', 'macos', '26'),
+        'safari260_ios': ImpersonateTarget('safari', '26.0', 'ios', '26.0'),
+    },
+}
+
+# Needed for curl_cffi < 0.11
+# See: https://github.com/lexiforest/curl_cffi/commit/d2f15c7a31506a08d217fcc04ae7570c39f5f5bb
+_TARGETS_COMPAT_LOOKUP = {
+    'safari153': 'safari15_3',
+    'safari155': 'safari15_5',
+    'safari170': 'safari17_0',
+    'safari172_ios': 'safari17_2_ios',
+    'safari180': 'safari18_0',
+    'safari180_ios': 'safari18_0_ios',
 }
 
 
@@ -158,16 +180,19 @@ class CurlCFFIRH(ImpersonateRequestHandler, InstanceStoreMixin):
     _SUPPORTED_FEATURES = (Features.NO_PROXY, Features.ALL_PROXY)
     _SUPPORTED_PROXY_SCHEMES = ('http', 'https', 'socks4', 'socks4a', 'socks5', 'socks5h')
     _SUPPORTED_IMPERSONATE_TARGET_MAP = {
-        target: name if curl_cffi_version >= (0, 9) else curl_cffi.requests.BrowserType[name]
-        for name, target in dict(sorted(itertools.chain.from_iterable(
+        target: (
+            name if curl_cffi_version >= (0, 11)
+            else _TARGETS_COMPAT_LOOKUP.get(name, name) if curl_cffi_version >= (0, 9)
+            else curl_cffi.requests.BrowserType[_TARGETS_COMPAT_LOOKUP.get(name, name)]
+        ) for name, target in dict(sorted(itertools.chain.from_iterable(
             targets.items()
             for version, targets in BROWSER_TARGETS.items()
             if curl_cffi_version >= version
         ), key=lambda x: (
             # deprioritize mobile targets since they give very different behavior
             x[1].os not in ('ios', 'android'),
-            # prioritize edge < firefox < safari < chrome
-            ('edge', 'firefox', 'safari', 'chrome').index(x[1].client),
+            # prioritize tor < edge < firefox < safari < chrome
+            ('tor', 'edge', 'firefox', 'safari', 'chrome').index(x[1].client),
             # prioritize newest version
             float(x[1].version) if x[1].version else 0,
             # group by os name

plugins/youtube_download/yt_dlp/networking/_helper.py

@@ -13,7 +13,6 @@ import urllib.request
 from .exceptions import RequestError
 from ..dependencies import certifi
 from ..socks import ProxyType, sockssocket
-from ..utils import format_field, traverse_obj
 
 if typing.TYPE_CHECKING:
     from collections.abc import Iterable
@@ -82,19 +81,6 @@ def make_socks_proxy_opts(socks_proxy):
     }
 
 
-def select_proxy(url, proxies):
-    """Unified proxy selector for all backends"""
-    url_components = urllib.parse.urlparse(url)
-    if 'no' in proxies:
-        hostport = url_components.hostname + format_field(url_components.port, None, ':%s')
-        if urllib.request.proxy_bypass_environment(hostport, {'no': proxies['no']}):
-            return
-        elif urllib.request.proxy_bypass(hostport):  # check system settings
-            return
-
-    return traverse_obj(proxies, url_components.scheme or 'http', 'all')
-
-
 def get_redirect_method(method, status):
     """Unified redirect method handling"""
 
@@ -214,7 +200,7 @@ def wrap_request_errors(func):
 
 
 def _socket_connect(ip_addr, timeout, source_address):
-    af, socktype, proto, canonname, sa = ip_addr
+    af, socktype, proto, _canonname, sa = ip_addr
     sock = socket.socket(af, socktype, proto)
     try:
         if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
@@ -229,7 +215,7 @@ def _socket_connect(ip_addr, timeout, source_address):
 
 
 def create_socks_proxy_socket(dest_addr, proxy_args, proxy_ip_addr, timeout, source_address):
-    af, socktype, proto, canonname, sa = proxy_ip_addr
+    af, socktype, proto, _canonname, sa = proxy_ip_addr
     sock = sockssocket(af, socktype, proto)
     try:
         connect_proxy_args = proxy_args.copy()

plugins/youtube_download/yt_dlp/networking/_requests.py

@@ -1,16 +1,14 @@
 from __future__ import annotations
 
-import contextlib
 import functools
 import http.client
 import logging
 import re
-import socket
 import warnings
 
 from ..dependencies import brotli, requests, urllib3
 from ..utils import bug_reports_message, int_or_none, variadic
-from ..utils.networking import normalize_url
+from ..utils.networking import normalize_url, select_proxy
 
 if requests is None:
     raise ImportError('requests module is not installed')
@@ -20,9 +18,9 @@ if urllib3 is None:
 
 urllib3_version = tuple(int_or_none(x, default=0) for x in urllib3.__version__.split('.'))
 
-if urllib3_version < (1, 26, 17):
+if urllib3_version < (2, 0, 2):
     urllib3._yt_dlp__version = f'{urllib3.__version__} (unsupported)'
-    raise ImportError('Only urllib3 >= 1.26.17 is supported')
+    raise ImportError('Only urllib3 >= 2.0.2 is supported')
 
 if requests.__build__ < 0x023202:
     requests._yt_dlp__version = f'{requests.__version__} (unsupported)'
@@ -41,7 +39,6 @@ from ._helper import (
     create_socks_proxy_socket,
     get_redirect_method,
     make_socks_proxy_opts,
-    select_proxy,
 )
 from .common import (
     Features,
@@ -102,27 +99,10 @@ class Urllib3PercentREOverride:
 # https://github.com/urllib3/urllib3/commit/a2697e7c6b275f05879b60f593c5854a816489f0
 import urllib3.util.url
 
-if hasattr(urllib3.util.url, 'PERCENT_RE'):
-    urllib3.util.url.PERCENT_RE = Urllib3PercentREOverride(urllib3.util.url.PERCENT_RE)
-elif hasattr(urllib3.util.url, '_PERCENT_RE'):  # urllib3 >= 2.0.0
+if hasattr(urllib3.util.url, '_PERCENT_RE'):  # was 'PERCENT_RE' in urllib3 < 2.0.0
     urllib3.util.url._PERCENT_RE = Urllib3PercentREOverride(urllib3.util.url._PERCENT_RE)
 else:
-    warnings.warn('Failed to patch PERCENT_RE in urllib3 (does the attribute exist?)' + bug_reports_message())
-
-'''
-Workaround for issue in urllib.util.ssl_.py: ssl_wrap_context does not pass
-server_hostname to SSLContext.wrap_socket if server_hostname is an IP,
-however this is an issue because we set check_hostname to True in our SSLContext.
-
-Monkey-patching IS_SECURETRANSPORT forces ssl_wrap_context to pass server_hostname regardless.
-
-This has been fixed in urllib3 2.0+.
-See: https://github.com/urllib3/urllib3/issues/517
-'''
-
-if urllib3_version < (2, 0, 0):
-    with contextlib.suppress(Exception):
-        urllib3.util.IS_SECURETRANSPORT = urllib3.util.ssl_.IS_SECURETRANSPORT = True
+    warnings.warn('Failed to patch _PERCENT_RE in urllib3 (does the attribute exist?)' + bug_reports_message())
 
 
 # Requests will not automatically handle no_proxy by default
@@ -141,6 +121,12 @@ class RequestsResponseAdapter(Response):
 
     def read(self, amt: int | None = None):
         try:
+            # Work around issue with `.read(amt)` then `.read()`
+            # See: https://github.com/urllib3/urllib3/issues/3636
+            if amt is None:
+                # compat: py3.9: Python 3.9 preallocates the whole read buffer, read in chunks
+                read_chunk = functools.partial(self.fp.read, 1 << 20, decode_content=True)
+                return b''.join(iter(read_chunk, b''))
             # Interact with urllib3 response directly.
             return self.fp.read(amt, decode_content=True)
 
@@ -308,7 +294,7 @@ class RequestsRH(RequestHandler, InstanceStoreMixin):
             max_retries=urllib3.util.retry.Retry(False),
         )
         session.adapters.clear()
-        session.headers = requests.models.CaseInsensitiveDict({'Connection': 'keep-alive'})
+        session.headers = requests.models.CaseInsensitiveDict()
         session.mount('https://', http_adapter)
         session.mount('http://', http_adapter)
         session.cookies = cookiejar
@@ -317,6 +303,7 @@ class RequestsRH(RequestHandler, InstanceStoreMixin):
 
     def _prepare_headers(self, _, headers):
         add_accept_encoding_header(headers, SUPPORTED_ENCODINGS)
+        headers.setdefault('Connection', 'keep-alive')
 
     def _send(self, request):
 
@@ -390,7 +377,7 @@ class SocksHTTPConnection(urllib3.connection.HTTPConnection):
                 source_address=self.source_address,
                 _create_socket_func=functools.partial(
                     create_socks_proxy_socket, (self.host, self.port), self._proxy_args))
-        except (socket.timeout, TimeoutError) as e:
+        except TimeoutError as e:
             raise urllib3.exceptions.ConnectTimeoutError(
                 self, f'Connection to {self.host} timed out. (connect timeout={self.timeout})') from e
         except SocksProxyError as e:

plugins/youtube_download/yt_dlp/networking/_urllib.py

@@ -26,7 +26,6 @@ from ._helper import (
     create_socks_proxy_socket,
     get_redirect_method,
     make_socks_proxy_opts,
-    select_proxy,
 )
 from .common import Features, RequestHandler, Response, register_rh
 from .exceptions import (
@@ -41,7 +40,7 @@ from .exceptions import (
 from ..dependencies import brotli
 from ..socks import ProxyError as SocksProxyError
 from ..utils import update_url_query
-from ..utils.networking import normalize_url
+from ..utils.networking import normalize_url, select_proxy
 
 SUPPORTED_ENCODINGS = ['gzip', 'deflate']
 CONTENT_DECODE_ERRORS = [zlib.error, OSError]

plugins/youtube_download/yt_dlp/networking/_websockets.py

@@ -11,8 +11,8 @@ from ._helper import (
     create_connection,
     create_socks_proxy_socket,
     make_socks_proxy_opts,
-    select_proxy,
 )
+from ..utils.networking import select_proxy
 from .common import Features, Response, register_rh
 from .exceptions import (
     CertificateVerifyError,

plugins/youtube_download/yt_dlp/networking/common.py

@@ -12,6 +12,7 @@ import urllib.response
 from collections.abc import Iterable, Mapping
 from email.message import Message
 from http import HTTPStatus
+from types import NoneType
 
 from ._helper import make_ssl_context, wrap_request_errors
 from .exceptions import (
@@ -20,7 +21,6 @@ from .exceptions import (
     TransportError,
     UnsupportedRequest,
 )
-from ..compat.types import NoneType
 from ..cookies import YoutubeDLCookieJar
 from ..utils import (
     bug_reports_message,

plugins/youtube_download/yt_dlp/networking/impersonate.py

@@ -3,11 +3,11 @@ from __future__ import annotations
 import re
 from abc import ABC
 from dataclasses import dataclass
+from types import NoneType
 from typing import Any
 
 from .common import RequestHandler, register_preference, Request
 from .exceptions import UnsupportedRequest
-from ..compat.types import NoneType
 from ..utils import classproperty, join_nonempty
 from ..utils.networking import std_headers, HTTPHeaderDict