Updated yt_dlp version; added extremly basic dumb cache setup in thumbnailer; moved build and script as well as deb folder to build
This commit is contained in:
@@ -42,6 +42,8 @@ from .globals import (
|
||||
plugin_pps,
|
||||
all_plugins_loaded,
|
||||
plugin_dirs,
|
||||
supported_js_runtimes,
|
||||
supported_remote_components,
|
||||
)
|
||||
from .minicurses import format_text
|
||||
from .networking import HEADRequest, Request, RequestDirector
|
||||
@@ -533,6 +535,18 @@ class YoutubeDL:
|
||||
See "EXTRACTOR ARGUMENTS" for details.
|
||||
Argument values must always be a list of string(s).
|
||||
E.g. {'youtube': {'skip': ['dash', 'hls']}}
|
||||
js_runtimes: A dictionary of JavaScript runtime keys (in lower case) to enable
|
||||
and a dictionary of additional configuration for the runtime.
|
||||
Currently supported runtimes are 'deno', 'node', 'bun', and 'quickjs'.
|
||||
If None, the default runtime of "deno" will be enabled.
|
||||
The runtime configuration dictionary can have the following keys:
|
||||
- path: Path to the executable (optional)
|
||||
E.g. {'deno': {'path': '/path/to/deno'}
|
||||
remote_components: A list of remote components that are allowed to be fetched when required.
|
||||
Supported components:
|
||||
- ejs:npm (external JavaScript components from npm)
|
||||
- ejs:github (external JavaScript components from yt-dlp-ejs GitHub)
|
||||
By default, no remote components are allowed to be fetched.
|
||||
mark_watched: Mark videos watched (even with --simulate). Only for YouTube
|
||||
|
||||
The following options are deprecated and may be removed in the future:
|
||||
@@ -581,7 +595,7 @@ class YoutubeDL:
|
||||
'width', 'height', 'asr', 'audio_channels', 'fps',
|
||||
'tbr', 'abr', 'vbr', 'filesize', 'filesize_approx',
|
||||
'timestamp', 'release_timestamp', 'available_at',
|
||||
'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
|
||||
'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count', 'save_count',
|
||||
'average_rating', 'comment_count', 'age_limit',
|
||||
'start_time', 'end_time',
|
||||
'chapter_number', 'season_number', 'episode_number',
|
||||
@@ -717,6 +731,13 @@ class YoutubeDL:
|
||||
else:
|
||||
raise
|
||||
|
||||
# Note: this must be after plugins are loaded
|
||||
self.params['js_runtimes'] = self.params.get('js_runtimes', {'deno': {}})
|
||||
self._clean_js_runtimes(self.params['js_runtimes'])
|
||||
|
||||
self.params['remote_components'] = set(self.params.get('remote_components', ()))
|
||||
self._clean_remote_components(self.params['remote_components'])
|
||||
|
||||
self.params['compat_opts'] = set(self.params.get('compat_opts', ()))
|
||||
self.params['http_headers'] = HTTPHeaderDict(std_headers, self.params.get('http_headers'))
|
||||
self._load_cookies(self.params['http_headers'].get('Cookie')) # compat
|
||||
@@ -829,6 +850,36 @@ class YoutubeDL:
|
||||
|
||||
self.archive = preload_download_archive(self.params.get('download_archive'))
|
||||
|
||||
def _clean_js_runtimes(self, runtimes):
|
||||
if not (
|
||||
isinstance(runtimes, dict)
|
||||
and all(isinstance(k, str) and (v is None or isinstance(v, dict)) for k, v in runtimes.items())
|
||||
):
|
||||
raise ValueError('Invalid js_runtimes format, expected a dict of {runtime: {config}}')
|
||||
|
||||
if unsupported_runtimes := runtimes.keys() - supported_js_runtimes.value.keys():
|
||||
self.report_warning(
|
||||
f'Ignoring unsupported JavaScript runtime(s): {", ".join(unsupported_runtimes)}.'
|
||||
f' Supported runtimes: {", ".join(supported_js_runtimes.value.keys())}.')
|
||||
for rt in unsupported_runtimes:
|
||||
runtimes.pop(rt)
|
||||
|
||||
def _clean_remote_components(self, remote_components: set):
|
||||
if unsupported_remote_components := set(remote_components) - set(supported_remote_components.value):
|
||||
self.report_warning(
|
||||
f'Ignoring unsupported remote component(s): {", ".join(unsupported_remote_components)}.'
|
||||
f' Supported remote components: {", ".join(supported_remote_components.value)}.')
|
||||
for rt in unsupported_remote_components:
|
||||
remote_components.remove(rt)
|
||||
|
||||
@functools.cached_property
|
||||
def _js_runtimes(self):
|
||||
runtimes = {}
|
||||
for name, config in self.params.get('js_runtimes', {}).items():
|
||||
runtime_cls = supported_js_runtimes.value.get(name)
|
||||
runtimes[name] = runtime_cls(path=config.get('path')) if runtime_cls else None
|
||||
return runtimes
|
||||
|
||||
def warn_if_short_id(self, argv):
|
||||
# short YouTube ID starting with dash?
|
||||
idxs = [
|
||||
@@ -2975,6 +3026,10 @@ class YoutubeDL:
|
||||
format_selector = self.format_selector
|
||||
while True:
|
||||
if interactive_format_selection:
|
||||
if not formats:
|
||||
# Bypass interactive format selection if no formats & --ignore-no-formats-error
|
||||
formats_to_download = None
|
||||
break
|
||||
req_format = input(self._format_screen('\nEnter format selector ', self.Styles.EMPHASIS)
|
||||
+ '(Press ENTER for default, or Ctrl+C to quit)'
|
||||
+ self._format_screen(': ', self.Styles.EMPHASIS))
|
||||
@@ -4064,6 +4119,18 @@ class YoutubeDL:
|
||||
join_nonempty(*get_package_info(m)) for m in available_dependencies.values()
|
||||
})) or 'none'))
|
||||
|
||||
if not self.params.get('js_runtimes'):
|
||||
write_debug('JS runtimes: none (disabled)')
|
||||
else:
|
||||
write_debug('JS runtimes: %s' % (', '.join(sorted(
|
||||
f'{name} (unknown)' if runtime is None
|
||||
else join_nonempty(
|
||||
runtime.info.name,
|
||||
runtime.info.version + (' (unsupported)' if runtime.info.supported is False else ''),
|
||||
)
|
||||
for name, runtime in self._js_runtimes.items() if runtime is None or runtime.info is not None
|
||||
)) or 'none'))
|
||||
|
||||
write_debug(f'Proxy map: {self.proxies}')
|
||||
write_debug(f'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers.values())}')
|
||||
|
||||
|
||||
@@ -61,8 +61,15 @@ from .utils import (
|
||||
shell_quote,
|
||||
variadic,
|
||||
write_string,
|
||||
|
||||
)
|
||||
from .utils._utils import _UnsafeExtensionError
|
||||
from .utils._jsruntime import (
|
||||
BunJsRuntime as _BunJsRuntime,
|
||||
DenoJsRuntime as _DenoJsRuntime,
|
||||
NodeJsRuntime as _NodeJsRuntime,
|
||||
QuickJsRuntime as _QuickJsRuntime,
|
||||
)
|
||||
from .YoutubeDL import YoutubeDL
|
||||
|
||||
|
||||
@@ -773,6 +780,10 @@ def parse_options(argv=None):
|
||||
else opts.audioformat if (opts.extractaudio and opts.audioformat in FFmpegExtractAudioPP.SUPPORTED_EXTS)
|
||||
else None)
|
||||
|
||||
js_runtimes = {
|
||||
runtime.lower(): {'path': path} for runtime, path in (
|
||||
[*arg.split(':', 1), None][:2] for arg in opts.js_runtimes)}
|
||||
|
||||
return ParsedOptions(parser, opts, urls, {
|
||||
'usenetrc': opts.usenetrc,
|
||||
'netrc_location': opts.netrc_location,
|
||||
@@ -940,6 +951,8 @@ def parse_options(argv=None):
|
||||
'geo_bypass_country': opts.geo_bypass_country,
|
||||
'geo_bypass_ip_block': opts.geo_bypass_ip_block,
|
||||
'useid': opts.useid or None,
|
||||
'js_runtimes': js_runtimes,
|
||||
'remote_components': opts.remote_components,
|
||||
'warn_when_outdated': opts.update_self is None,
|
||||
'_warnings': warnings,
|
||||
'_deprecation_warnings': deprecation_warnings,
|
||||
@@ -1081,6 +1094,16 @@ def main(argv=None):
|
||||
|
||||
from .extractor import gen_extractors, list_extractors
|
||||
|
||||
# Register JS runtimes and remote components
|
||||
from .globals import supported_js_runtimes, supported_remote_components
|
||||
supported_js_runtimes.value['deno'] = _DenoJsRuntime
|
||||
supported_js_runtimes.value['node'] = _NodeJsRuntime
|
||||
supported_js_runtimes.value['bun'] = _BunJsRuntime
|
||||
supported_js_runtimes.value['quickjs'] = _QuickJsRuntime
|
||||
|
||||
supported_remote_components.value.append('ejs:github')
|
||||
supported_remote_components.value.append('ejs:npm')
|
||||
|
||||
__all__ = [
|
||||
'YoutubeDL',
|
||||
'gen_extractors',
|
||||
|
||||
@@ -34,3 +34,4 @@ print(f'Adding imports: {hiddenimports}')
|
||||
excludedimports = ['youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts', 'bundle']
|
||||
|
||||
datas = collect_data_files('curl_cffi', includes=['cacert.pem'])
|
||||
datas += collect_data_files('yt_dlp_ejs', includes=['**/*.js'])
|
||||
|
||||
@@ -125,7 +125,7 @@ def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(),
|
||||
|
||||
|
||||
def _extract_firefox_cookies(profile, container, logger):
|
||||
MAX_SUPPORTED_DB_SCHEMA_VERSION = 16
|
||||
MAX_SUPPORTED_DB_SCHEMA_VERSION = 17
|
||||
|
||||
logger.info('Extracting cookies from firefox')
|
||||
if not sqlite3:
|
||||
@@ -166,6 +166,8 @@ def _extract_firefox_cookies(profile, container, logger):
|
||||
db_schema_version = cursor.execute('PRAGMA user_version;').fetchone()[0]
|
||||
if db_schema_version > MAX_SUPPORTED_DB_SCHEMA_VERSION:
|
||||
logger.warning(f'Possibly unsupported firefox cookies database version: {db_schema_version}')
|
||||
else:
|
||||
logger.debug(f'Firefox cookies database version: {db_schema_version}')
|
||||
if isinstance(container_id, int):
|
||||
logger.debug(
|
||||
f'Only loading cookies from firefox container "{container}", ID {container_id}')
|
||||
@@ -210,9 +212,16 @@ def _firefox_browser_dirs():
|
||||
|
||||
else:
|
||||
yield from map(os.path.expanduser, (
|
||||
# New installations of FF147+ respect the XDG base directory specification
|
||||
# Ref: https://bugzilla.mozilla.org/show_bug.cgi?id=259356
|
||||
os.path.join(_config_home(), 'mozilla/firefox'),
|
||||
# Existing FF version<=146 installations
|
||||
'~/.mozilla/firefox',
|
||||
'~/snap/firefox/common/.mozilla/firefox',
|
||||
# Flatpak XDG: https://docs.flatpak.org/en/latest/conventions.html#xdg-base-directories
|
||||
'~/.var/app/org.mozilla.firefox/config/mozilla/firefox',
|
||||
'~/.var/app/org.mozilla.firefox/.mozilla/firefox',
|
||||
# Snap installations do not respect the XDG base directory specification
|
||||
'~/snap/firefox/common/.mozilla/firefox',
|
||||
))
|
||||
|
||||
|
||||
@@ -557,7 +566,7 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor):
|
||||
|
||||
|
||||
def _extract_safari_cookies(profile, logger):
|
||||
if sys.platform != 'darwin':
|
||||
if sys.platform not in ('darwin', 'ios'):
|
||||
raise ValueError(f'unsupported platform: {sys.platform}')
|
||||
|
||||
if profile:
|
||||
|
||||
@@ -81,6 +81,12 @@ except ImportError:
|
||||
|
||||
from . import Cryptodome
|
||||
|
||||
try:
|
||||
import yt_dlp_ejs
|
||||
except ImportError:
|
||||
yt_dlp_ejs = None
|
||||
|
||||
|
||||
all_dependencies = {k: v for k, v in globals().items() if not k.startswith('_')}
|
||||
available_dependencies = {k: v for k, v in all_dependencies.items() if v}
|
||||
|
||||
|
||||
@@ -461,7 +461,8 @@ class FileDownloader:
|
||||
min_sleep_interval = self.params.get('sleep_interval') or 0
|
||||
max_sleep_interval = self.params.get('max_sleep_interval') or 0
|
||||
|
||||
if available_at := info_dict.get('available_at'):
|
||||
requested_formats = info_dict.get('requested_formats') or [info_dict]
|
||||
if available_at := max(f.get('available_at') or 0 for f in requested_formats):
|
||||
forced_sleep_interval = available_at - int(time.time())
|
||||
if forced_sleep_interval > min_sleep_interval:
|
||||
sleep_note = 'as required by the site'
|
||||
|
||||
@@ -457,6 +457,8 @@ class FFmpegFD(ExternalFD):
|
||||
|
||||
@classmethod
|
||||
def available(cls, path=None):
|
||||
# TODO: Fix path for ffmpeg
|
||||
# Fixme: This may be wrong when --ffmpeg-location is used
|
||||
return FFmpegPostProcessor().available
|
||||
|
||||
def on_process_started(self, proc, stdin):
|
||||
@@ -488,20 +490,6 @@ class FFmpegFD(ExternalFD):
|
||||
if not self.params.get('verbose'):
|
||||
args += ['-hide_banner']
|
||||
|
||||
args += traverse_obj(info_dict, ('downloader_options', 'ffmpeg_args', ...))
|
||||
|
||||
# These exists only for compatibility. Extractors should use
|
||||
# info_dict['downloader_options']['ffmpeg_args'] instead
|
||||
args += info_dict.get('_ffmpeg_args') or []
|
||||
seekable = info_dict.get('_seekable')
|
||||
if seekable is not None:
|
||||
# setting -seekable prevents ffmpeg from guessing if the server
|
||||
# supports seeking(by adding the header `Range: bytes=0-`), which
|
||||
# can cause problems in some cases
|
||||
# https://github.com/ytdl-org/youtube-dl/issues/11800#issuecomment-275037127
|
||||
# http://trac.ffmpeg.org/ticket/6125#comment:10
|
||||
args += ['-seekable', '1' if seekable else '0']
|
||||
|
||||
env = None
|
||||
proxy = self.params.get('proxy')
|
||||
if proxy:
|
||||
@@ -521,39 +509,10 @@ class FFmpegFD(ExternalFD):
|
||||
env['HTTP_PROXY'] = proxy
|
||||
env['http_proxy'] = proxy
|
||||
|
||||
protocol = info_dict.get('protocol')
|
||||
|
||||
if protocol == 'rtmp':
|
||||
player_url = info_dict.get('player_url')
|
||||
page_url = info_dict.get('page_url')
|
||||
app = info_dict.get('app')
|
||||
play_path = info_dict.get('play_path')
|
||||
tc_url = info_dict.get('tc_url')
|
||||
flash_version = info_dict.get('flash_version')
|
||||
live = info_dict.get('rtmp_live', False)
|
||||
conn = info_dict.get('rtmp_conn')
|
||||
if player_url is not None:
|
||||
args += ['-rtmp_swfverify', player_url]
|
||||
if page_url is not None:
|
||||
args += ['-rtmp_pageurl', page_url]
|
||||
if app is not None:
|
||||
args += ['-rtmp_app', app]
|
||||
if play_path is not None:
|
||||
args += ['-rtmp_playpath', play_path]
|
||||
if tc_url is not None:
|
||||
args += ['-rtmp_tcurl', tc_url]
|
||||
if flash_version is not None:
|
||||
args += ['-rtmp_flashver', flash_version]
|
||||
if live:
|
||||
args += ['-rtmp_live', 'live']
|
||||
if isinstance(conn, list):
|
||||
for entry in conn:
|
||||
args += ['-rtmp_conn', entry]
|
||||
elif isinstance(conn, str):
|
||||
args += ['-rtmp_conn', conn]
|
||||
|
||||
start_time, end_time = info_dict.get('section_start') or 0, info_dict.get('section_end')
|
||||
|
||||
fallback_input_args = traverse_obj(info_dict, ('downloader_options', 'ffmpeg_args', ...))
|
||||
|
||||
selected_formats = info_dict.get('requested_formats') or [info_dict]
|
||||
for i, fmt in enumerate(selected_formats):
|
||||
is_http = re.match(r'https?://', fmt['url'])
|
||||
@@ -572,6 +531,44 @@ class FFmpegFD(ExternalFD):
|
||||
if end_time:
|
||||
args += ['-t', str(end_time - start_time)]
|
||||
|
||||
protocol = fmt.get('protocol')
|
||||
|
||||
if protocol == 'rtmp':
|
||||
player_url = fmt.get('player_url')
|
||||
page_url = fmt.get('page_url')
|
||||
app = fmt.get('app')
|
||||
play_path = fmt.get('play_path')
|
||||
tc_url = fmt.get('tc_url')
|
||||
flash_version = fmt.get('flash_version')
|
||||
live = fmt.get('rtmp_live', False)
|
||||
conn = fmt.get('rtmp_conn')
|
||||
if player_url is not None:
|
||||
args += ['-rtmp_swfverify', player_url]
|
||||
if page_url is not None:
|
||||
args += ['-rtmp_pageurl', page_url]
|
||||
if app is not None:
|
||||
args += ['-rtmp_app', app]
|
||||
if play_path is not None:
|
||||
args += ['-rtmp_playpath', play_path]
|
||||
if tc_url is not None:
|
||||
args += ['-rtmp_tcurl', tc_url]
|
||||
if flash_version is not None:
|
||||
args += ['-rtmp_flashver', flash_version]
|
||||
if live:
|
||||
args += ['-rtmp_live', 'live']
|
||||
if isinstance(conn, list):
|
||||
for entry in conn:
|
||||
args += ['-rtmp_conn', entry]
|
||||
elif isinstance(conn, str):
|
||||
args += ['-rtmp_conn', conn]
|
||||
|
||||
elif protocol == 'http_dash_segments' and info_dict.get('is_live'):
|
||||
# ffmpeg may try to read past the latest available segments for
|
||||
# live DASH streams unless we pass `-re`. In modern ffmpeg, this
|
||||
# is an alias of `-readrate 1`, but `-readrate` was not added
|
||||
# until ffmpeg 5.0, so we must stick to using `-re`
|
||||
args += ['-re']
|
||||
|
||||
url = fmt['url']
|
||||
if self.params.get('enable_file_urls') and url.startswith('file:'):
|
||||
# The default protocol_whitelist is 'file,crypto,data' when reading local m3u8 URLs,
|
||||
@@ -586,6 +583,7 @@ class FFmpegFD(ExternalFD):
|
||||
# https://trac.ffmpeg.org/ticket/2702
|
||||
url = re.sub(r'^file://(?:localhost)?/', 'file:' if os.name == 'nt' else 'file:/', url)
|
||||
|
||||
args += traverse_obj(fmt, ('downloader_options', 'ffmpeg_args', ...)) or fallback_input_args
|
||||
args += [*self._configuration_args((f'_i{i + 1}', '_i')), '-i', url]
|
||||
|
||||
if not (start_time or end_time) or not self.params.get('force_keyframes_at_cuts'):
|
||||
|
||||
@@ -75,6 +75,7 @@ from .afreecatv import (
|
||||
AfreecaTVLiveIE,
|
||||
AfreecaTVUserIE,
|
||||
)
|
||||
from .agalega import AGalegaIE
|
||||
from .agora import (
|
||||
TokFMAuditionIE,
|
||||
TokFMPodcastIE,
|
||||
@@ -83,6 +84,7 @@ from .agora import (
|
||||
)
|
||||
from .airtv import AirTVIE
|
||||
from .aitube import AitubeKZVideoIE
|
||||
from .alibaba import AlibabaIE
|
||||
from .aliexpress import AliExpressLiveIE
|
||||
from .aljazeera import AlJazeeraIE
|
||||
from .allocine import AllocineIE
|
||||
@@ -143,6 +145,8 @@ from .archiveorg import (
|
||||
from .arcpublishing import ArcPublishingIE
|
||||
from .ard import (
|
||||
ARDIE,
|
||||
ARDAudiothekIE,
|
||||
ARDAudiothekPlaylistIE,
|
||||
ARDBetaMediathekIE,
|
||||
ARDMediathekCollectionIE,
|
||||
)
|
||||
@@ -266,6 +270,7 @@ from .bitchute import (
|
||||
BitChuteChannelIE,
|
||||
BitChuteIE,
|
||||
)
|
||||
from .bitmovin import BitmovinIE
|
||||
from .blackboardcollaborate import (
|
||||
BlackboardCollaborateIE,
|
||||
BlackboardCollaborateLaunchIE,
|
||||
@@ -426,6 +431,7 @@ from .cpac import (
|
||||
)
|
||||
from .cracked import CrackedIE
|
||||
from .craftsy import CraftsyIE
|
||||
from .croatianfilm import CroatianFilmIE
|
||||
from .crooksandliars import CrooksAndLiarsIE
|
||||
from .crowdbunker import (
|
||||
CrowdBunkerChannelIE,
|
||||
@@ -633,12 +639,16 @@ from .fc2 import (
|
||||
)
|
||||
from .fczenit import FczenitIE
|
||||
from .fifa import FifaIE
|
||||
from .filmarchiv import FilmArchivIE
|
||||
from .filmon import (
|
||||
FilmOnChannelIE,
|
||||
FilmOnIE,
|
||||
)
|
||||
from .filmweb import FilmwebIE
|
||||
from .firsttv import FirstTVIE
|
||||
from .firsttv import (
|
||||
FirstTVIE,
|
||||
FirstTVLiveIE,
|
||||
)
|
||||
from .fivetv import FiveTVIE
|
||||
from .flextv import FlexTVIE
|
||||
from .flickr import FlickrIE
|
||||
@@ -685,6 +695,10 @@ from .frontendmasters import (
|
||||
FrontendMastersIE,
|
||||
FrontendMastersLessonIE,
|
||||
)
|
||||
from .frontro import (
|
||||
TheChosenGroupIE,
|
||||
TheChosenIE,
|
||||
)
|
||||
from .fujitv import FujiTVFODPlus7IE
|
||||
from .funk import FunkIE
|
||||
from .funker530 import Funker530IE
|
||||
@@ -1074,11 +1088,6 @@ from .mangomolo import (
|
||||
MangomoloLiveIE,
|
||||
MangomoloVideoIE,
|
||||
)
|
||||
from .manoto import (
|
||||
ManotoTVIE,
|
||||
ManotoTVLiveIE,
|
||||
ManotoTVShowIE,
|
||||
)
|
||||
from .manyvids import ManyVidsIE
|
||||
from .maoritv import MaoriTVIE
|
||||
from .markiza import (
|
||||
@@ -1088,7 +1097,10 @@ from .markiza import (
|
||||
from .massengeschmacktv import MassengeschmackTVIE
|
||||
from .masters import MastersIE
|
||||
from .matchtv import MatchTVIE
|
||||
from .mave import MaveIE
|
||||
from .mave import (
|
||||
MaveChannelIE,
|
||||
MaveIE,
|
||||
)
|
||||
from .mbn import MBNIE
|
||||
from .mdr import MDRIE
|
||||
from .medaltv import MedalTVIE
|
||||
@@ -1195,6 +1207,7 @@ from .musicdex import (
|
||||
MusicdexPlaylistIE,
|
||||
MusicdexSongIE,
|
||||
)
|
||||
from .mux import MuxIE
|
||||
from .mx3 import (
|
||||
Mx3IE,
|
||||
Mx3NeoIE,
|
||||
@@ -1216,6 +1229,7 @@ from .n1 import (
|
||||
N1InfoAssetIE,
|
||||
N1InfoIIE,
|
||||
)
|
||||
from .nascar import NascarClassicsIE
|
||||
from .nate import (
|
||||
NateIE,
|
||||
NateProgramIE,
|
||||
@@ -1261,6 +1275,7 @@ from .nebula import (
|
||||
NebulaChannelIE,
|
||||
NebulaClassIE,
|
||||
NebulaIE,
|
||||
NebulaSeasonIE,
|
||||
NebulaSubscriptionsIE,
|
||||
)
|
||||
from .nekohacker import NekoHackerIE
|
||||
@@ -1269,6 +1284,10 @@ from .nest import (
|
||||
NestClipIE,
|
||||
NestIE,
|
||||
)
|
||||
from .netapp import (
|
||||
NetAppCollectionIE,
|
||||
NetAppVideoIE,
|
||||
)
|
||||
from .neteasemusic import (
|
||||
NetEaseMusicAlbumIE,
|
||||
NetEaseMusicDjRadioIE,
|
||||
@@ -1291,12 +1310,6 @@ from .newgrounds import (
|
||||
)
|
||||
from .newspicks import NewsPicksIE
|
||||
from .newsy import NewsyIE
|
||||
from .nextmedia import (
|
||||
AppleDailyIE,
|
||||
NextMediaActionNewsIE,
|
||||
NextMediaIE,
|
||||
NextTVIE,
|
||||
)
|
||||
from .nexx import (
|
||||
NexxEmbedIE,
|
||||
NexxIE,
|
||||
@@ -1361,6 +1374,7 @@ from .nova import (
|
||||
NovaIE,
|
||||
)
|
||||
from .novaplay import NovaPlayIE
|
||||
from .nowcanal import NowCanalIE
|
||||
from .nowness import (
|
||||
NownessIE,
|
||||
NownessPlaylistIE,
|
||||
@@ -1464,6 +1478,7 @@ from .palcomp3 import (
|
||||
PalcoMP3IE,
|
||||
PalcoMP3VideoIE,
|
||||
)
|
||||
from .pandatv import PandaTvIE
|
||||
from .panopto import (
|
||||
PanoptoIE,
|
||||
PanoptoListIE,
|
||||
@@ -1812,10 +1827,6 @@ from .scrippsnetworks import (
|
||||
ScrippsNetworksWatchIE,
|
||||
)
|
||||
from .scrolller import ScrolllerIE
|
||||
from .scte import (
|
||||
SCTEIE,
|
||||
SCTECourseIE,
|
||||
)
|
||||
from .sejmpl import SejmIE
|
||||
from .sen import SenIE
|
||||
from .senalcolombia import SenalColombiaLiveIE
|
||||
@@ -1997,6 +2008,11 @@ from .taptap import (
|
||||
TapTapMomentIE,
|
||||
TapTapPostIntlIE,
|
||||
)
|
||||
from .tarangplus import (
|
||||
TarangPlusEpisodesIE,
|
||||
TarangPlusPlaylistIE,
|
||||
TarangPlusVideoIE,
|
||||
)
|
||||
from .tass import TassIE
|
||||
from .tbs import TBSIE
|
||||
from .tbsjp import (
|
||||
@@ -2514,6 +2530,7 @@ from .yappy import (
|
||||
YappyIE,
|
||||
YappyProfileIE,
|
||||
)
|
||||
from .yfanefa import YfanefaIE
|
||||
from .yle_areena import YleAreenaIE
|
||||
from .youjizz import YouJizzIE
|
||||
from .youku import (
|
||||
|
||||
@@ -321,6 +321,8 @@ class ABCIViewIE(InfoExtractor):
|
||||
entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
|
||||
if formats:
|
||||
break
|
||||
else:
|
||||
formats = []
|
||||
|
||||
subtitles = {}
|
||||
src_vtt = stream.get('captions', {}).get('src-vtt')
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import json
|
||||
import time
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import jwt_decode_hs256, url_or_none
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class AGalegaBaseIE(InfoExtractor):
|
||||
_access_token = None
|
||||
|
||||
@staticmethod
|
||||
def _jwt_is_expired(token):
|
||||
return jwt_decode_hs256(token)['exp'] - time.time() < 120
|
||||
|
||||
def _refresh_access_token(self, video_id):
|
||||
AGalegaBaseIE._access_token = self._download_json(
|
||||
'https://www.agalega.gal/api/fetch-api/jwt/token', video_id,
|
||||
note='Downloading access token',
|
||||
data=json.dumps({
|
||||
'username': None,
|
||||
'password': None,
|
||||
'client': 'crtvg',
|
||||
'checkExistsCookies': False,
|
||||
}).encode())['access']
|
||||
|
||||
def _call_api(self, endpoint, display_id, note, fatal=True, query=None):
|
||||
if not AGalegaBaseIE._access_token or self._jwt_is_expired(AGalegaBaseIE._access_token):
|
||||
self._refresh_access_token(endpoint)
|
||||
return self._download_json(
|
||||
f'https://api-agalega.interactvty.com/api/2.0/contents/{endpoint}', display_id,
|
||||
note=note, fatal=fatal, query=query,
|
||||
headers={'Authorization': f'jwtok {AGalegaBaseIE._access_token}'})
|
||||
|
||||
|
||||
class AGalegaIE(AGalegaBaseIE):
|
||||
IE_NAME = 'agalega:videos'
|
||||
_VALID_URL = r'https?://(?:www\.)?agalega\.gal/videos/(?:detail/)?(?P<id>[0-9]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.agalega.gal/videos/288664-lr-ninguencheconta',
|
||||
'md5': '04533a66c5f863d08dd9724b11d1c223',
|
||||
'info_dict': {
|
||||
'id': '288664',
|
||||
'title': 'Roberto e Ángel Martín atenden consultas dos espectadores',
|
||||
'description': 'O cómico ademais fai un repaso dalgúns momentos da súa traxectoria profesional',
|
||||
'thumbnail': 'https://crtvg-bucket.flumotion.cloud/content_cards/2ef32c3b9f6249d9868fd8f11d389d3d.png',
|
||||
'ext': 'mp4',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.agalega.gal/videos/detail/296152-pulso-activo-7',
|
||||
'md5': '26df7fdcf859f38ad92d837279d6b56d',
|
||||
'info_dict': {
|
||||
'id': '296152',
|
||||
'title': 'Pulso activo | 18-11-2025',
|
||||
'description': 'Anxo, Noemí, Silvia e Estrella comparten as sensacións da clase de Eddy.',
|
||||
'thumbnail': 'https://crtvg-bucket.flumotion.cloud/content_cards/a6bb7da6c8994b82bf961ac6cad1707b.png',
|
||||
'ext': 'mp4',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
content_data = self._call_api(
|
||||
f'content/{video_id}/', video_id, note='Downloading content data', fatal=False,
|
||||
query={
|
||||
'optional_fields': 'image,is_premium,short_description,has_subtitle',
|
||||
})
|
||||
resource_data = self._call_api(
|
||||
f'content_resources/{video_id}/', video_id, note='Downloading resource data',
|
||||
query={
|
||||
'optional_fields': 'media_url',
|
||||
})
|
||||
|
||||
formats = []
|
||||
subtitles = {}
|
||||
for m3u8_url in traverse_obj(resource_data, ('results', ..., 'media_url', {url_or_none})):
|
||||
fmts, subs = self._extract_m3u8_formats_and_subtitles(
|
||||
m3u8_url, video_id, ext='mp4', m3u8_id='hls')
|
||||
formats.extend(fmts)
|
||||
self._merge_subtitles(subs, target=subtitles)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
**traverse_obj(content_data, {
|
||||
'title': ('name', {str}),
|
||||
'description': (('description', 'short_description'), {str}, any),
|
||||
'thumbnail': ('image', {url_or_none}),
|
||||
}),
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import int_or_none, str_or_none, url_or_none
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class AlibabaIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?alibaba\.com/product-detail/[\w-]+_(?P<id>\d+)\.html'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.alibaba.com/product-detail/Kids-Entertainment-Bouncer-Bouncy-Castle-Waterslide_1601271126969.html',
|
||||
'info_dict': {
|
||||
'id': '6000280444270',
|
||||
'display_id': '1601271126969',
|
||||
'ext': 'mp4',
|
||||
'title': 'Kids Entertainment Bouncer Bouncy Castle Waterslide Juex Gonflables Commercial Inflatable Tropical Water Slide',
|
||||
'duration': 30,
|
||||
'thumbnail': 'https://sc04.alicdn.com/kf/Hc5bb391974454af18c7a4f91cbe4062bg.jpg_120x120.jpg',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
product_data = self._search_json(
|
||||
r'window\.detailData\s*=', webpage, 'detail data', display_id)['globalData']['product']
|
||||
|
||||
return {
|
||||
**traverse_obj(product_data, ('mediaItems', lambda _, v: v['type'] == 'video' and v['videoId'], any, {
|
||||
'id': ('videoId', {int}, {str_or_none}),
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'thumbnail': ('videoCoverUrl', {url_or_none}),
|
||||
'formats': ('videoUrl', lambda _, v: url_or_none(v['videoUrl']), {
|
||||
'url': 'videoUrl',
|
||||
'format_id': ('definition', {str_or_none}),
|
||||
'tbr': ('bitrate', {int_or_none}),
|
||||
'width': ('width', {int_or_none}),
|
||||
'height': ('height', {int_or_none}),
|
||||
'filesize': ('length', {int_or_none}),
|
||||
}),
|
||||
})),
|
||||
'title': traverse_obj(product_data, ('subject', {str})),
|
||||
'display_id': display_id,
|
||||
}
|
||||
@@ -5,12 +5,9 @@ import re
|
||||
import urllib.parse
|
||||
|
||||
from .common import InfoExtractor
|
||||
from .youtube import YoutubeBaseInfoExtractor, YoutubeIE
|
||||
from ..networking import HEADRequest
|
||||
from ..networking.exceptions import HTTPError
|
||||
from .youtube import YoutubeBaseInfoExtractor
|
||||
from ..utils import (
|
||||
KNOWN_EXTENSIONS,
|
||||
ExtractorError,
|
||||
bug_reports_message,
|
||||
clean_html,
|
||||
dict_get,
|
||||
@@ -21,18 +18,14 @@ from ..utils import (
|
||||
join_nonempty,
|
||||
js_to_json,
|
||||
merge_dicts,
|
||||
mimetype2ext,
|
||||
orderedSet,
|
||||
parse_duration,
|
||||
parse_qs,
|
||||
str_or_none,
|
||||
str_to_int,
|
||||
traverse_obj,
|
||||
try_get,
|
||||
unified_strdate,
|
||||
unified_timestamp,
|
||||
url_or_none,
|
||||
urlhandle_detect_ext,
|
||||
)
|
||||
|
||||
|
||||
@@ -286,7 +279,7 @@ class ArchiveOrgIE(InfoExtractor):
|
||||
'url': 'https://archive.org/' + track['file'].lstrip('/'),
|
||||
}
|
||||
|
||||
metadata = self._download_json('http://archive.org/metadata/' + identifier, identifier)
|
||||
metadata = self._download_json(f'https://archive.org/metadata/{identifier}', identifier)
|
||||
m = metadata['metadata']
|
||||
identifier = m['identifier']
|
||||
|
||||
@@ -471,7 +464,7 @@ class YoutubeWebArchiveIE(InfoExtractor):
|
||||
'url': 'https://web.archive.org/web/20110712231407/http://www.youtube.com/watch?v=lTx3G6h2xyA',
|
||||
'info_dict': {
|
||||
'id': 'lTx3G6h2xyA',
|
||||
'ext': 'flv',
|
||||
'ext': 'mp4',
|
||||
'title': 'Madeon - Pop Culture (live mashup)',
|
||||
'upload_date': '20110711',
|
||||
'uploader': 'Madeon',
|
||||
@@ -578,7 +571,7 @@ class YoutubeWebArchiveIE(InfoExtractor):
|
||||
'url': 'https://web.archive.org/web/20110126141719/http://www.youtube.com/watch?v=Q_yjX80U7Yc',
|
||||
'info_dict': {
|
||||
'id': 'Q_yjX80U7Yc',
|
||||
'ext': 'flv',
|
||||
'ext': 'webm',
|
||||
'title': 'Spray Paint Art by Clay Butler: Purple Fantasy Forest',
|
||||
'uploader_id': 'claybutlermusic',
|
||||
'description': 'md5:4595264559e3d0a0ceb3f011f6334543',
|
||||
@@ -680,6 +673,55 @@ class YoutubeWebArchiveIE(InfoExtractor):
|
||||
'upload_date': '20120407',
|
||||
'uploader_id': 'thecomputernerd01',
|
||||
},
|
||||
}, {
|
||||
# Contains split audio/video formats
|
||||
'url': 'ytarchive:o_T_S_TU12M',
|
||||
'info_dict': {
|
||||
'id': 'o_T_S_TU12M',
|
||||
'ext': 'mp4',
|
||||
'title': 'Prairie Pulse 1218; Lin Enger, Paul Olson',
|
||||
'description': 'md5:36e7a34cdc8508e35a920ec042e799c7',
|
||||
'uploader': 'Prairie Public',
|
||||
'channel_id': 'UC4BOzQel6tvJm7OEDd3vZlw',
|
||||
'channel_url': 'https://www.youtube.com/channel/UC4BOzQel6tvJm7OEDd3vZlw',
|
||||
'duration': 1606,
|
||||
'upload_date': '20150213',
|
||||
},
|
||||
}, {
|
||||
# Video unavailable through wayback-fakeurl
|
||||
'url': 'ytarchive:SQCom7wjGDs',
|
||||
'info_dict': {
|
||||
'id': 'SQCom7wjGDs',
|
||||
'ext': 'mp4',
|
||||
'title': 'Jamin Warren from PBS Game/Show decides that Portal is a feminist Game [Top Hats and No Brain]',
|
||||
'description': 'md5:c0cb876dd075483ead9afcc86798efb0',
|
||||
'uploader': 'Top Hats and Champagne',
|
||||
'uploader_id': 'sparrowtm',
|
||||
'uploader_url': 'https://www.youtube.com/user/sparrowtm',
|
||||
'channel_id': 'UCW3T5nG4iEkI7HjG-Du3HQA',
|
||||
'channel_url': 'https://www.youtube.com/channel/UCW3T5nG4iEkI7HjG-Du3HQA',
|
||||
'duration': 1500,
|
||||
'thumbnail': 'https://web.archive.org/web/20160108040020if_/https://i.ytimg.com/vi/SQCom7wjGDs/maxresdefault.jpg',
|
||||
'upload_date': '20160107',
|
||||
},
|
||||
}, {
|
||||
# dmuxed formats
|
||||
'url': 'https://web.archive.org/web/20240922160632/https://www.youtube.com/watch?v=z7hzvTL3k1k',
|
||||
'info_dict': {
|
||||
'id': 'z7hzvTL3k1k',
|
||||
'ext': 'webm',
|
||||
'title': 'Praise the Lord and Pass the Ammunition (BARRXN REMIX)',
|
||||
'description': 'md5:45dbf2c71c23b0734c8dfb82dd1e94b6',
|
||||
'uploader': 'Barrxn',
|
||||
'uploader_id': 'TheRockstar6086',
|
||||
'uploader_url': 'https://www.youtube.com/user/TheRockstar6086',
|
||||
'channel_id': 'UCjJPGUTtvR9uizmawn2ThqA',
|
||||
'channel_url': 'https://www.youtube.com/channel/UCjJPGUTtvR9uizmawn2ThqA',
|
||||
'duration': 125,
|
||||
'thumbnail': r're:https?://.*\.(jpg|webp)',
|
||||
'upload_date': '20201207',
|
||||
},
|
||||
'params': {'format': 'bv'},
|
||||
}, {
|
||||
'url': 'https://web.archive.org/web/http://www.youtube.com/watch?v=kH-G_aIBlFw',
|
||||
'only_matching': True,
|
||||
@@ -724,6 +766,113 @@ class YoutubeWebArchiveIE(InfoExtractor):
|
||||
_OLDEST_CAPTURE_DATE = 20050214000000
|
||||
_NEWEST_CAPTURE_DATE = 20500101000000
|
||||
|
||||
_FORMATS = {
|
||||
'5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'vcodec': 'h263'},
|
||||
'6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'vcodec': 'h263'},
|
||||
'13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
|
||||
'17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'vcodec': 'mp4v'},
|
||||
'18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
'22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
'34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
'35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
# itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
|
||||
'36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
|
||||
'37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
'38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
'43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'vcodec': 'vp8'},
|
||||
'44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'vcodec': 'vp8'},
|
||||
'45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'vcodec': 'vp8'},
|
||||
'46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'vcodec': 'vp8'},
|
||||
'59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
'78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
|
||||
|
||||
# 3D videos
|
||||
'82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'vcodec': 'h264', 'preference': -20},
|
||||
'83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'vcodec': 'h264', 'preference': -20},
|
||||
'84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'vcodec': 'h264', 'preference': -20},
|
||||
'85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'vcodec': 'h264', 'preference': -20},
|
||||
'100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'vcodec': 'vp8', 'preference': -20},
|
||||
'101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'vcodec': 'vp8', 'preference': -20},
|
||||
'102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'vcodec': 'vp8', 'preference': -20},
|
||||
|
||||
# Apple HTTP Live Streaming
|
||||
'91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
'92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
'93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
'94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
'95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
'96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
'132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
'151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'vcodec': 'h264'},
|
||||
|
||||
# DASH mp4 video
|
||||
'133': {'ext': 'mp4', 'height': 240, 'vcodec': 'h264', 'acodec': 'none'},
|
||||
'134': {'ext': 'mp4', 'height': 360, 'vcodec': 'h264', 'acodec': 'none'},
|
||||
'135': {'ext': 'mp4', 'height': 480, 'vcodec': 'h264', 'acodec': 'none'},
|
||||
'136': {'ext': 'mp4', 'height': 720, 'vcodec': 'h264', 'acodec': 'none'},
|
||||
'137': {'ext': 'mp4', 'height': 1080, 'vcodec': 'h264', 'acodec': 'none'},
|
||||
'138': {'ext': 'mp4', 'vcodec': 'h264', 'acodec': 'none'}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
|
||||
'160': {'ext': 'mp4', 'height': 144, 'vcodec': 'h264', 'acodec': 'none'},
|
||||
'212': {'ext': 'mp4', 'height': 480, 'vcodec': 'h264', 'acodec': 'none'},
|
||||
'264': {'ext': 'mp4', 'height': 1440, 'vcodec': 'h264', 'acodec': 'none'},
|
||||
'298': {'ext': 'mp4', 'height': 720, 'vcodec': 'h264', 'fps': 60, 'acodec': 'none'},
|
||||
'299': {'ext': 'mp4', 'height': 1080, 'vcodec': 'h264', 'fps': 60, 'acodec': 'none'},
|
||||
'266': {'ext': 'mp4', 'height': 2160, 'vcodec': 'h264', 'acodec': 'none'},
|
||||
|
||||
# Dash mp4 audio
|
||||
'139': {'ext': 'm4a', 'acodec': 'aac', 'vcodec': 'none'},
|
||||
'140': {'ext': 'm4a', 'acodec': 'aac', 'vcodec': 'none'},
|
||||
'141': {'ext': 'm4a', 'acodec': 'aac', 'vcodec': 'none'},
|
||||
'256': {'ext': 'm4a', 'acodec': 'aac', 'vcodec': 'none'},
|
||||
'258': {'ext': 'm4a', 'acodec': 'aac', 'vcodec': 'none'},
|
||||
'325': {'ext': 'm4a', 'acodec': 'dtse', 'vcodec': 'none'},
|
||||
'328': {'ext': 'm4a', 'acodec': 'ec-3', 'vcodec': 'none'},
|
||||
|
||||
# Dash webm
|
||||
'167': {'ext': 'webm', 'height': 360, 'width': 640, 'vcodec': 'vp8'},
|
||||
'168': {'ext': 'webm', 'height': 480, 'width': 854, 'vcodec': 'vp8'},
|
||||
'169': {'ext': 'webm', 'height': 720, 'width': 1280, 'vcodec': 'vp8'},
|
||||
'170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'vcodec': 'vp8'},
|
||||
'218': {'ext': 'webm', 'height': 480, 'width': 854, 'vcodec': 'vp8'},
|
||||
'219': {'ext': 'webm', 'height': 480, 'width': 854, 'vcodec': 'vp8'},
|
||||
'278': {'ext': 'webm', 'height': 144, 'vcodec': 'vp9', 'acodec': 'none'},
|
||||
'242': {'ext': 'webm', 'height': 240, 'vcodec': 'vp9', 'acodec': 'none'},
|
||||
'243': {'ext': 'webm', 'height': 360, 'vcodec': 'vp9', 'acodec': 'none'},
|
||||
'244': {'ext': 'webm', 'height': 480, 'vcodec': 'vp9', 'acodec': 'none'},
|
||||
'245': {'ext': 'webm', 'height': 480, 'vcodec': 'vp9', 'acodec': 'none'},
|
||||
'246': {'ext': 'webm', 'height': 480, 'vcodec': 'vp9', 'acodec': 'none'},
|
||||
'247': {'ext': 'webm', 'height': 720, 'vcodec': 'vp9', 'acodec': 'none'},
|
||||
'248': {'ext': 'webm', 'height': 1080, 'vcodec': 'vp9', 'acodec': 'none'},
|
||||
'271': {'ext': 'webm', 'height': 1440, 'vcodec': 'vp9', 'acodec': 'none'},
|
||||
# itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
|
||||
'272': {'ext': 'webm', 'height': 2160, 'vcodec': 'vp9', 'acodec': 'none'},
|
||||
'302': {'ext': 'webm', 'height': 720, 'vcodec': 'vp9', 'fps': 60, 'acodec': 'none'},
|
||||
'303': {'ext': 'webm', 'height': 1080, 'vcodec': 'vp9', 'fps': 60, 'acodec': 'none'},
|
||||
'308': {'ext': 'webm', 'height': 1440, 'vcodec': 'vp9', 'fps': 60, 'acodec': 'none'},
|
||||
'313': {'ext': 'webm', 'height': 2160, 'vcodec': 'vp9', 'acodec': 'none'},
|
||||
'315': {'ext': 'webm', 'height': 2160, 'vcodec': 'vp9', 'fps': 60, 'acodec': 'none'},
|
||||
|
||||
# Dash webm audio
|
||||
'171': {'ext': 'webm', 'acodec': 'vorbis', 'vcodec': 'none'},
|
||||
'172': {'ext': 'webm', 'acodec': 'vorbis', 'vcodec': 'none'},
|
||||
|
||||
# Dash webm audio with opus inside
|
||||
'249': {'ext': 'webm', 'acodec': 'opus', 'vcodec': 'none'},
|
||||
'250': {'ext': 'webm', 'acodec': 'opus', 'vcodec': 'none'},
|
||||
'251': {'ext': 'webm', 'acodec': 'opus', 'vcodec': 'none'},
|
||||
|
||||
# av01 video only formats sometimes served with "unknown" codecs
|
||||
'394': {'ext': 'mp4', 'height': 144, 'vcodec': 'av01.0.00M.08', 'acodec': 'none'},
|
||||
'395': {'ext': 'mp4', 'height': 240, 'vcodec': 'av01.0.00M.08', 'acodec': 'none'},
|
||||
'396': {'ext': 'mp4', 'height': 360, 'vcodec': 'av01.0.01M.08', 'acodec': 'none'},
|
||||
'397': {'ext': 'mp4', 'height': 480, 'vcodec': 'av01.0.04M.08', 'acodec': 'none'},
|
||||
'398': {'ext': 'mp4', 'height': 720, 'vcodec': 'av01.0.05M.08', 'acodec': 'none'},
|
||||
'399': {'ext': 'mp4', 'height': 1080, 'vcodec': 'av01.0.08M.08', 'acodec': 'none'},
|
||||
'400': {'ext': 'mp4', 'height': 1440, 'vcodec': 'av01.0.12M.08', 'acodec': 'none'},
|
||||
'401': {'ext': 'mp4', 'height': 2160, 'vcodec': 'av01.0.12M.08', 'acodec': 'none'},
|
||||
}
|
||||
|
||||
def _call_cdx_api(self, item_id, url, filters: list | None = None, collapse: list | None = None, query: dict | None = None, note=None, fatal=False):
|
||||
# CDX docs: https://github.com/internetarchive/wayback/blob/master/wayback-cdx-server/README.md
|
||||
query = {
|
||||
@@ -929,27 +1078,30 @@ class YoutubeWebArchiveIE(InfoExtractor):
|
||||
capture_dates.extend([self._OLDEST_CAPTURE_DATE, self._NEWEST_CAPTURE_DATE])
|
||||
return orderedSet(filter(None, capture_dates))
|
||||
|
||||
def _parse_fmt(self, fmt, extra_info=None):
|
||||
format_id = traverse_obj(fmt, ('url', {parse_qs}, 'itag', 0))
|
||||
return {
|
||||
'format_id': format_id,
|
||||
**self._FORMATS.get(format_id, {}),
|
||||
**traverse_obj(fmt, {
|
||||
'url': ('url', {lambda x: f'https://web.archive.org/web/2id_/{x}'}),
|
||||
'ext': ('ext', {str}),
|
||||
'filesize': ('url', {parse_qs}, 'clen', 0, {int_or_none}),
|
||||
}),
|
||||
**(extra_info or {}),
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id, url_date, url_date_2 = self._match_valid_url(url).group('id', 'date', 'date2')
|
||||
url_date = url_date or url_date_2
|
||||
|
||||
urlh = None
|
||||
retry_manager = self.RetryManager(fatal=False)
|
||||
for retry in retry_manager:
|
||||
try:
|
||||
urlh = self._request_webpage(
|
||||
HEADRequest(f'https://web.archive.org/web/2oe_/http://wayback-fakeurl.archive.org/yt/{video_id}'),
|
||||
video_id, note='Fetching archived video file url', expected_status=True)
|
||||
except ExtractorError as e:
|
||||
# HTTP Error 404 is expected if the video is not saved.
|
||||
if isinstance(e.cause, HTTPError) and e.cause.status == 404:
|
||||
self.raise_no_formats(
|
||||
'The requested video is not archived, indexed, or there is an issue with web.archive.org (try again later)', expected=True)
|
||||
else:
|
||||
retry.error = e
|
||||
video_info = self._download_json(
|
||||
'https://web.archive.org/__wb/videoinfo', video_id,
|
||||
query={'vtype': 'youtube', 'vid': video_id})
|
||||
|
||||
if retry_manager.error:
|
||||
self.raise_no_formats(retry_manager.error, expected=True, video_id=video_id)
|
||||
if not traverse_obj(video_info, 'formats'):
|
||||
self.raise_no_formats(
|
||||
'The requested video is not archived or indexed', expected=True)
|
||||
|
||||
capture_dates = self._get_capture_dates(video_id, int_or_none(url_date))
|
||||
self.write_debug('Captures to try: ' + join_nonempty(*capture_dates, delim=', '))
|
||||
@@ -968,25 +1120,15 @@ class YoutubeWebArchiveIE(InfoExtractor):
|
||||
|
||||
info['thumbnails'] = self._extract_thumbnails(video_id)
|
||||
|
||||
if urlh:
|
||||
url = urllib.parse.unquote(urlh.url)
|
||||
video_file_url_qs = parse_qs(url)
|
||||
# Attempt to recover any ext & format info from playback url & response headers
|
||||
fmt = {'url': url, 'filesize': int_or_none(urlh.headers.get('x-archive-orig-content-length'))}
|
||||
itag = try_get(video_file_url_qs, lambda x: x['itag'][0])
|
||||
if itag and itag in YoutubeIE._formats:
|
||||
fmt.update(YoutubeIE._formats[itag])
|
||||
fmt.update({'format_id': itag})
|
||||
else:
|
||||
mime = try_get(video_file_url_qs, lambda x: x['mime'][0])
|
||||
ext = (mimetype2ext(mime)
|
||||
or urlhandle_detect_ext(urlh)
|
||||
or mimetype2ext(urlh.headers.get('x-archive-guessed-content-type')))
|
||||
fmt.update({'ext': ext})
|
||||
info['formats'] = [fmt]
|
||||
if not info.get('duration'):
|
||||
info['duration'] = str_to_int(try_get(video_file_url_qs, lambda x: x['dur'][0]))
|
||||
formats = []
|
||||
if video_info.get('dmux'):
|
||||
for vf in traverse_obj(video_info, ('formats', 'video', lambda _, v: url_or_none(v['url']))):
|
||||
formats.append(self._parse_fmt(vf, {'acodec': 'none'}))
|
||||
for af in traverse_obj(video_info, ('formats', 'audio', lambda _, v: url_or_none(v['url']))):
|
||||
formats.append(self._parse_fmt(af, {'vcodec': 'none'}))
|
||||
else:
|
||||
for fmt in traverse_obj(video_info, ('formats', lambda _, v: url_or_none(v['url']))):
|
||||
formats.append(self._parse_fmt(fmt))
|
||||
info['formats'] = formats
|
||||
|
||||
if not info.get('title'):
|
||||
info['title'] = video_id
|
||||
return info
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import functools
|
||||
import json
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
@@ -15,11 +16,12 @@ from ..utils import (
|
||||
remove_start,
|
||||
str_or_none,
|
||||
unified_strdate,
|
||||
update_url,
|
||||
update_url_query,
|
||||
url_or_none,
|
||||
xpath_text,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
from ..utils.traversal import traverse_obj, value
|
||||
|
||||
|
||||
class ARDMediathekBaseIE(InfoExtractor):
|
||||
@@ -601,3 +603,163 @@ class ARDMediathekCollectionIE(InfoExtractor):
|
||||
return self.playlist_result(
|
||||
OnDemandPagedList(fetch_page, self._PAGE_SIZE), full_id, display_id=display_id,
|
||||
title=page_data.get('title'), description=page_data.get('synopsis'))
|
||||
|
||||
|
||||
class ARDAudiothekBaseIE(InfoExtractor):
|
||||
def _graphql_query(self, urn, query):
|
||||
return self._download_json(
|
||||
'https://api.ardaudiothek.de/graphql', urn,
|
||||
data=json.dumps({
|
||||
'query': query,
|
||||
'variables': {'id': urn},
|
||||
}).encode(), headers={
|
||||
'Content-Type': 'application/json',
|
||||
})['data']
|
||||
|
||||
|
||||
class ARDAudiothekIE(ARDAudiothekBaseIE):
|
||||
_VALID_URL = r'https:?//(?:www\.)?ardaudiothek\.de/episode/(?P<id>urn:ard:(?:episode|section|extra):[a-f0-9]{16})'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://www.ardaudiothek.de/episode/urn:ard:episode:eabead1add170e93/',
|
||||
'info_dict': {
|
||||
'id': 'urn:ard:episode:eabead1add170e93',
|
||||
'ext': 'mp3',
|
||||
'upload_date': '20240717',
|
||||
'duration': 3339,
|
||||
'title': 'CAIMAN CLUB (S04E04): Cash Out',
|
||||
'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:ed64411a07a4b405',
|
||||
'description': 'md5:0e5d127a3832ae59e8bab40a91a5dadc',
|
||||
'display_id': 'urn:ard:episode:eabead1add170e93',
|
||||
'timestamp': 1721181641,
|
||||
'series': '1LIVE Caiman Club',
|
||||
'channel': 'WDR',
|
||||
'episode': 'Episode 4',
|
||||
'episode_number': 4,
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.ardaudiothek.de/episode/urn:ard:section:855c7a53dac72e0a/',
|
||||
'info_dict': {
|
||||
'id': 'urn:ard:section:855c7a53dac72e0a',
|
||||
'ext': 'mp4',
|
||||
'upload_date': '20241231',
|
||||
'duration': 3304,
|
||||
'title': 'Illegaler DDR-Detektiv: Doberschütz und die letzte Staatsjagd (1/2) - Wendezeit',
|
||||
'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:b9b4f1e8b93da4dd',
|
||||
'description': 'md5:3552d571e1959754cff66c1da6c0fdae',
|
||||
'display_id': 'urn:ard:section:855c7a53dac72e0a',
|
||||
'timestamp': 1735629900,
|
||||
'series': 'Auf der Spur – Die ARD Ermittlerkrimis',
|
||||
'channel': 'ARD',
|
||||
'episode': 'Episode 1',
|
||||
'episode_number': 1,
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.ardaudiothek.de/episode/urn:ard:extra:d2fe7303d2dcbf5d/',
|
||||
'info_dict': {
|
||||
'id': 'urn:ard:extra:d2fe7303d2dcbf5d',
|
||||
'ext': 'mp3',
|
||||
'title': 'Trailer: Fanta Vier Forever, Baby!?!',
|
||||
'description': 'md5:b64a586f2e976b8bb5ea0a79dbd8751c',
|
||||
'channel': 'SWR',
|
||||
'duration': 62,
|
||||
'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:48d3c255969be803',
|
||||
'series': 'Fanta Vier Forever, Baby!?!',
|
||||
'timestamp': 1732108217,
|
||||
'upload_date': '20241120',
|
||||
},
|
||||
}]
|
||||
|
||||
_QUERY_ITEM = '''\
|
||||
query($id: ID!) {
|
||||
item(id: $id) {
|
||||
audioList {
|
||||
href
|
||||
distributionType
|
||||
audioBitrate
|
||||
audioCodec
|
||||
}
|
||||
show {
|
||||
title
|
||||
}
|
||||
image {
|
||||
url1X1
|
||||
}
|
||||
programSet {
|
||||
publicationService {
|
||||
organizationName
|
||||
}
|
||||
}
|
||||
description
|
||||
title
|
||||
duration
|
||||
startDate
|
||||
episodeNumber
|
||||
}
|
||||
}'''
|
||||
|
||||
def _real_extract(self, url):
|
||||
urn = self._match_id(url)
|
||||
item = self._graphql_query(urn, self._QUERY_ITEM)['item']
|
||||
return {
|
||||
'id': urn,
|
||||
**traverse_obj(item, {
|
||||
'formats': ('audioList', lambda _, v: url_or_none(v['href']), {
|
||||
'url': 'href',
|
||||
'format_id': ('distributionType', {str}),
|
||||
'abr': ('audioBitrate', {int_or_none}),
|
||||
'acodec': ('audioCodec', {str}),
|
||||
'vcodec': {value('none')},
|
||||
}),
|
||||
'channel': ('programSet', 'publicationService', 'organizationName', {str}),
|
||||
'description': ('description', {str}),
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'series': ('show', 'title', {str}),
|
||||
'episode_number': ('episodeNumber', {int_or_none}),
|
||||
'thumbnail': ('image', 'url1X1', {url_or_none}, {update_url(query=None)}),
|
||||
'timestamp': ('startDate', {parse_iso8601}),
|
||||
'title': ('title', {str}),
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
class ARDAudiothekPlaylistIE(ARDAudiothekBaseIE):
|
||||
_VALID_URL = r'https:?//(?:www\.)?ardaudiothek\.de/sendung/(?P<playlist>[\w-]+)/(?P<id>urn:ard:show:[a-f0-9]{16})'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://www.ardaudiothek.de/sendung/mia-insomnia/urn:ard:show:c405aa26d9a4060a/',
|
||||
'info_dict': {
|
||||
'display_id': 'mia-insomnia',
|
||||
'title': 'Mia Insomnia',
|
||||
'id': 'urn:ard:show:c405aa26d9a4060a',
|
||||
'description': 'md5:d9ceb7a6b4d26a4db3316573bb564292',
|
||||
},
|
||||
'playlist_mincount': 37,
|
||||
}, {
|
||||
'url': 'https://www.ardaudiothek.de/sendung/100-berlin/urn:ard:show:4d248e0806ce37bc/',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
_QUERY_PLAYLIST = '''
|
||||
query($id: ID!) {
|
||||
show(id: $id) {
|
||||
title
|
||||
description
|
||||
items(filter: { isPublished: { equalTo: true } }) {
|
||||
nodes {
|
||||
url
|
||||
}
|
||||
}
|
||||
}
|
||||
}'''
|
||||
|
||||
def _real_extract(self, url):
|
||||
urn, playlist = self._match_valid_url(url).group('id', 'playlist')
|
||||
playlist_info = self._graphql_query(urn, self._QUERY_PLAYLIST)['show']
|
||||
entries = []
|
||||
for url in traverse_obj(playlist_info, ('items', 'nodes', ..., 'url', {url_or_none})):
|
||||
entries.append(self.url_result(url, ie=ARDAudiothekIE))
|
||||
return self.playlist_result(entries, urn, display_id=playlist, **traverse_obj(playlist_info, {
|
||||
'title': ('title', {str}),
|
||||
'description': ('description', {str}),
|
||||
}))
|
||||
|
||||
@@ -5,16 +5,18 @@ import time
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
KNOWN_EXTENSIONS,
|
||||
ExtractorError,
|
||||
clean_html,
|
||||
extract_attributes,
|
||||
float_or_none,
|
||||
format_field,
|
||||
int_or_none,
|
||||
join_nonempty,
|
||||
parse_filesize,
|
||||
parse_qs,
|
||||
str_or_none,
|
||||
strftime_or_none,
|
||||
try_get,
|
||||
unified_strdate,
|
||||
unified_timestamp,
|
||||
update_url_query,
|
||||
url_or_none,
|
||||
@@ -411,70 +413,67 @@ class BandcampAlbumIE(BandcampIE): # XXX: Do not subclass from concrete IE
|
||||
|
||||
class BandcampWeeklyIE(BandcampIE): # XXX: Do not subclass from concrete IE
|
||||
IE_NAME = 'Bandcamp:weekly'
|
||||
_VALID_URL = r'https?://(?:www\.)?bandcamp\.com/?\?(?:.*?&)?show=(?P<id>\d+)'
|
||||
_VALID_URL = r'https?://(?:www\.)?bandcamp\.com/radio/?\?(?:[^#]+&)?show=(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://bandcamp.com/?show=224',
|
||||
'url': 'https://bandcamp.com/radio?show=224',
|
||||
'md5': '61acc9a002bed93986b91168aa3ab433',
|
||||
'info_dict': {
|
||||
'id': '224',
|
||||
'ext': 'mp3',
|
||||
'title': 'BC Weekly April 4th 2017 - Magic Moments',
|
||||
'title': 'Bandcamp Weekly, 2017-04-04',
|
||||
'description': 'md5:5d48150916e8e02d030623a48512c874',
|
||||
'duration': 5829.77,
|
||||
'release_date': '20170404',
|
||||
'thumbnail': 'https://f4.bcbits.com/img/9982549_0.jpg',
|
||||
'series': 'Bandcamp Weekly',
|
||||
'episode': 'Magic Moments',
|
||||
'episode_id': '224',
|
||||
'release_timestamp': 1491264000,
|
||||
'release_date': '20170404',
|
||||
'duration': 5829.77,
|
||||
},
|
||||
'params': {
|
||||
'format': 'mp3-128',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://bandcamp.com/?blah/blah@&show=228',
|
||||
'url': 'https://bandcamp.com/radio/?foo=bar&show=224',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
show_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, show_id)
|
||||
audio_data = self._download_json(
|
||||
'https://bandcamp.com/api/bcradio_api/1/get_show',
|
||||
show_id, 'Downloading radio show JSON',
|
||||
data=json.dumps({'id': show_id}).encode(),
|
||||
headers={'Content-Type': 'application/json'})['radioShowAudio']
|
||||
|
||||
blob = self._extract_data_attr(webpage, show_id, 'blob')
|
||||
stream_url = audio_data['streamUrl']
|
||||
format_id = traverse_obj(stream_url, ({parse_qs}, 'enc', -1))
|
||||
encoding, _, bitrate_str = (format_id or '').partition('-')
|
||||
|
||||
show = blob['bcw_data'][show_id]
|
||||
webpage = self._download_webpage(url, show_id, fatal=False)
|
||||
metadata = traverse_obj(
|
||||
self._extract_data_attr(webpage, show_id, 'blob', fatal=False),
|
||||
('appData', 'shows', lambda _, v: str(v['showId']) == show_id, any)) or {}
|
||||
|
||||
formats = []
|
||||
for format_id, format_url in show['audio_stream'].items():
|
||||
if not url_or_none(format_url):
|
||||
continue
|
||||
for known_ext in KNOWN_EXTENSIONS:
|
||||
if known_ext in format_id:
|
||||
ext = known_ext
|
||||
break
|
||||
else:
|
||||
ext = None
|
||||
formats.append({
|
||||
'format_id': format_id,
|
||||
'url': format_url,
|
||||
'ext': ext,
|
||||
'vcodec': 'none',
|
||||
})
|
||||
|
||||
title = show.get('audio_title') or 'Bandcamp Weekly'
|
||||
subtitle = show.get('subtitle')
|
||||
if subtitle:
|
||||
title += f' - {subtitle}'
|
||||
series_title = audio_data.get('title') or metadata.get('title')
|
||||
release_timestamp = unified_timestamp(audio_data.get('date')) or unified_timestamp(metadata.get('date'))
|
||||
|
||||
return {
|
||||
'id': show_id,
|
||||
'title': title,
|
||||
'description': show.get('desc') or show.get('short_desc'),
|
||||
'duration': float_or_none(show.get('audio_duration')),
|
||||
'is_live': False,
|
||||
'release_date': unified_strdate(show.get('published_date')),
|
||||
'series': 'Bandcamp Weekly',
|
||||
'episode': show.get('subtitle'),
|
||||
'episode_id': show_id,
|
||||
'formats': formats,
|
||||
'title': join_nonempty(series_title, strftime_or_none(release_timestamp, '%Y-%m-%d'), delim=', '),
|
||||
'series': series_title,
|
||||
'thumbnail': format_field(metadata, 'imageId', 'https://f4.bcbits.com/img/%s_0.jpg', default=None),
|
||||
'description': metadata.get('desc') or metadata.get('short_desc'),
|
||||
'duration': float_or_none(audio_data.get('duration')),
|
||||
'release_timestamp': release_timestamp,
|
||||
'formats': [{
|
||||
'url': stream_url,
|
||||
'format_id': format_id,
|
||||
'ext': encoding or 'mp3',
|
||||
'acodec': encoding or None,
|
||||
'vcodec': 'none',
|
||||
'abr': int_or_none(bitrate_str),
|
||||
}],
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import ExtractorError, urlencode_postdata
|
||||
from ..utils import ExtractorError, UserNotLive, urlencode_postdata
|
||||
|
||||
|
||||
class BigoIE(InfoExtractor):
|
||||
@@ -40,7 +40,7 @@ class BigoIE(InfoExtractor):
|
||||
info = info_raw.get('data') or {}
|
||||
|
||||
if not info.get('alive'):
|
||||
raise ExtractorError('This user is offline.', expected=True)
|
||||
raise UserNotLive(video_id=user_id)
|
||||
|
||||
formats, subs = self._extract_m3u8_formats_and_subtitles(
|
||||
info.get('hls_src'), user_id, 'mp4', 'm3u8')
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class BitmovinIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://streams\.bitmovin\.com/(?P<id>\w+)'
|
||||
_EMBED_REGEX = [r'<iframe\b[^>]+\bsrc=["\'](?P<url>(?:https?:)?//streams\.bitmovin\.com/(?P<id>\w+)[^"\']+)']
|
||||
_TESTS = [{
|
||||
'url': 'https://streams.bitmovin.com/cqkl1t5giv3lrce7pjbg/embed',
|
||||
'info_dict': {
|
||||
'id': 'cqkl1t5giv3lrce7pjbg',
|
||||
'ext': 'mp4',
|
||||
'title': 'Developing Osteopathic Residents as Faculty',
|
||||
'thumbnail': 'https://streams.bitmovin.com/cqkl1t5giv3lrce7pjbg/poster',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}, {
|
||||
'url': 'https://streams.bitmovin.com/cgl9rh94uvs51rqc8jhg/share',
|
||||
'info_dict': {
|
||||
'id': 'cgl9rh94uvs51rqc8jhg',
|
||||
'ext': 'mp4',
|
||||
'title': 'Big Buck Bunny (Streams Docs)',
|
||||
'thumbnail': 'https://streams.bitmovin.com/cgl9rh94uvs51rqc8jhg/poster',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}]
|
||||
_WEBPAGE_TESTS = [{
|
||||
# bitmovin-stream web component
|
||||
'url': 'https://www.institutionalinvestor.com/article/2bsw1in1l9k68mp9kritc/video-war-stories-over-board-games/best-case-i-get-fired-war-stories',
|
||||
'info_dict': {
|
||||
'id': 'cuiumeil6g115lc4li3g',
|
||||
'ext': 'mp4',
|
||||
'title': '[media] War Stories over Board Games: “Best Case: I Get Fired” ',
|
||||
'thumbnail': 'https://streams.bitmovin.com/cuiumeil6g115lc4li3g/poster',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}, {
|
||||
# iframe embed
|
||||
'url': 'https://www.clearblueionizer.com/en/pool-ionizers/mineral-pool-vs-saltwater-pool/',
|
||||
'info_dict': {
|
||||
'id': 'cvpvfsm1pf7itg7cfvtg',
|
||||
'ext': 'mp4',
|
||||
'title': 'Pool Ionizer vs. Salt Chlorinator',
|
||||
'thumbnail': 'https://streams.bitmovin.com/cvpvfsm1pf7itg7cfvtg/poster',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}]
|
||||
|
||||
@classmethod
|
||||
def _extract_embed_urls(cls, url, webpage):
|
||||
yield from super()._extract_embed_urls(url, webpage)
|
||||
for stream_id in re.findall(r'<bitmovin-stream\b[^>]*\bstream-id=["\'](?P<id>\w+)', webpage):
|
||||
yield f'https://streams.bitmovin.com/{stream_id}'
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
player_config = self._download_json(
|
||||
f'https://streams.bitmovin.com/{video_id}/config', video_id)['sources']
|
||||
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||
player_config['hls'], video_id, 'mp4')
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
**traverse_obj(player_config, {
|
||||
'title': ('title', {str}),
|
||||
'thumbnail': ('poster', {str}),
|
||||
}),
|
||||
}
|
||||
@@ -16,7 +16,7 @@ from ..utils.traversal import find_element, traverse_obj
|
||||
|
||||
|
||||
class BunnyCdnIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:iframe\.mediadelivery\.net|video\.bunnycdn\.com)/(?:embed|play)/(?P<library_id>\d+)/(?P<id>[\da-f-]+)'
|
||||
_VALID_URL = r'https?://(?:(?:iframe|player)\.mediadelivery\.net|video\.bunnycdn\.com)/(?:embed|play)/(?P<library_id>\d+)/(?P<id>[\da-f-]+)'
|
||||
_EMBED_REGEX = [rf'<iframe[^>]+src=[\'"](?P<url>{_VALID_URL}[^\'"]*)[\'"]']
|
||||
_TESTS = [{
|
||||
'url': 'https://iframe.mediadelivery.net/embed/113933/e73edec1-e381-4c8b-ae73-717a140e0924',
|
||||
@@ -39,7 +39,7 @@ class BunnyCdnIE(InfoExtractor):
|
||||
'timestamp': 1691145748,
|
||||
'thumbnail': r're:^https?://.*\.b-cdn\.net/32e34c4b-0d72-437c-9abb-05e67657da34/thumbnail_9172dc16\.jpg',
|
||||
'duration': 106.0,
|
||||
'description': 'md5:981a3e899a5c78352b21ed8b2f1efd81',
|
||||
'description': 'md5:11452bcb31f379ee3eaf1234d3264e44',
|
||||
'upload_date': '20230804',
|
||||
'title': 'Sanela ist Teil der #arbeitsmarktkraft',
|
||||
},
|
||||
@@ -58,6 +58,23 @@ class BunnyCdnIE(InfoExtractor):
|
||||
'thumbnail': r're:^https?://.*\.b-cdn\.net/2e8545ec-509d-4571-b855-4cf0235ccd75/thumbnail\.jpg',
|
||||
},
|
||||
'params': {'skip_download': True},
|
||||
}, {
|
||||
# Requires any Referer
|
||||
'url': 'https://iframe.mediadelivery.net/embed/289162/6372f5a3-68df-4ef7-a115-e1110186c477',
|
||||
'info_dict': {
|
||||
'id': '6372f5a3-68df-4ef7-a115-e1110186c477',
|
||||
'ext': 'mp4',
|
||||
'title': '12-Creating Small Asset Blockouts -Timelapse.mp4',
|
||||
'description': '',
|
||||
'duration': 263.0,
|
||||
'timestamp': 1724485440,
|
||||
'upload_date': '20240824',
|
||||
'thumbnail': r're:^https?://.*\.b-cdn\.net/6372f5a3-68df-4ef7-a115-e1110186c477/thumbnail\.jpg',
|
||||
},
|
||||
'params': {'skip_download': True},
|
||||
}, {
|
||||
'url': 'https://player.mediadelivery.net/embed/519128/875880a9-bcc2-4038-9e05-e5024bba9b70',
|
||||
'only_matching': True,
|
||||
}]
|
||||
_WEBPAGE_TESTS = [{
|
||||
# Stream requires Referer
|
||||
@@ -100,7 +117,7 @@ class BunnyCdnIE(InfoExtractor):
|
||||
video_id, library_id = self._match_valid_url(url).group('id', 'library_id')
|
||||
webpage = self._download_webpage(
|
||||
f'https://iframe.mediadelivery.net/embed/{library_id}/{video_id}', video_id,
|
||||
headers=traverse_obj(smuggled_data, {'Referer': 'Referer'}),
|
||||
headers={'Referer': smuggled_data.get('Referer') or 'https://iframe.mediadelivery.net/'},
|
||||
query=traverse_obj(parse_qs(url), {'token': 'token', 'expires': 'expires'}))
|
||||
|
||||
if html_title := self._html_extract_title(webpage, default=None) == '403':
|
||||
|
||||
@@ -27,7 +27,7 @@ from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class CDAIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:(?:www\.)?cda\.pl/video|ebd\.cda\.pl/[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
|
||||
_VALID_URL = r'https?://(?:(?:(?:www|m)\.)?cda\.pl/video|ebd\.cda\.pl/[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
|
||||
_NETRC_MACHINE = 'cdapl'
|
||||
|
||||
_BASE_URL = 'https://www.cda.pl'
|
||||
@@ -110,6 +110,9 @@ class CDAIE(InfoExtractor):
|
||||
}, {
|
||||
'url': 'http://ebd.cda.pl/0x0/5749950c',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://m.cda.pl/video/617297677',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _download_age_confirm_page(self, url, video_id, *args, **kwargs):
|
||||
@@ -367,35 +370,35 @@ class CDAIE(InfoExtractor):
|
||||
|
||||
class CDAFolderIE(InfoExtractor):
|
||||
_MAX_PAGE_SIZE = 36
|
||||
_VALID_URL = r'https?://(?:www\.)?cda\.pl/(?P<channel>[\w-]+)/folder/(?P<id>\d+)'
|
||||
_TESTS = [
|
||||
{
|
||||
'url': 'https://www.cda.pl/domino264/folder/31188385',
|
||||
'info_dict': {
|
||||
'id': '31188385',
|
||||
'title': 'SERIA DRUGA',
|
||||
},
|
||||
'playlist_mincount': 13,
|
||||
_VALID_URL = r'https?://(?:(?:www|m)\.)?cda\.pl/(?P<channel>[\w-]+)/folder/(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.cda.pl/domino264/folder/31188385',
|
||||
'info_dict': {
|
||||
'id': '31188385',
|
||||
'title': 'SERIA DRUGA',
|
||||
},
|
||||
{
|
||||
'url': 'https://www.cda.pl/smiechawaTV/folder/2664592/vfilm',
|
||||
'info_dict': {
|
||||
'id': '2664592',
|
||||
'title': 'VideoDowcipy - wszystkie odcinki',
|
||||
},
|
||||
'playlist_mincount': 71,
|
||||
'playlist_mincount': 13,
|
||||
}, {
|
||||
'url': 'https://www.cda.pl/smiechawaTV/folder/2664592/vfilm',
|
||||
'info_dict': {
|
||||
'id': '2664592',
|
||||
'title': 'VideoDowcipy - wszystkie odcinki',
|
||||
},
|
||||
{
|
||||
'url': 'https://www.cda.pl/DeliciousBeauty/folder/19129979/vfilm',
|
||||
'info_dict': {
|
||||
'id': '19129979',
|
||||
'title': 'TESTY KOSMETYKÓW',
|
||||
},
|
||||
'playlist_mincount': 139,
|
||||
}, {
|
||||
'url': 'https://www.cda.pl/FILMY-SERIALE-ANIME-KRESKOWKI-BAJKI/folder/18493422',
|
||||
'only_matching': True,
|
||||
}]
|
||||
'playlist_mincount': 71,
|
||||
}, {
|
||||
'url': 'https://www.cda.pl/DeliciousBeauty/folder/19129979/vfilm',
|
||||
'info_dict': {
|
||||
'id': '19129979',
|
||||
'title': 'TESTY KOSMETYKÓW',
|
||||
},
|
||||
'playlist_mincount': 139,
|
||||
}, {
|
||||
'url': 'https://www.cda.pl/FILMY-SERIALE-ANIME-KRESKOWKI-BAJKI/folder/18493422',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://m.cda.pl/smiechawaTV/folder/2664592/vfilm',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
folder_id, channel = self._match_valid_url(url).group('id', 'channel')
|
||||
|
||||
@@ -348,6 +348,7 @@ class InfoExtractor:
|
||||
duration: Length of the video in seconds, as an integer or float.
|
||||
view_count: How many users have watched the video on the platform.
|
||||
concurrent_view_count: How many users are currently watching the video on the platform.
|
||||
save_count: Number of times the video has been saved or bookmarked
|
||||
like_count: Number of positive ratings of the video
|
||||
dislike_count: Number of negative ratings of the video
|
||||
repost_count: Number of reposts of the video
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
from .common import InfoExtractor
|
||||
from .vimeo import VimeoIE
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
join_nonempty,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class CroatianFilmIE(InfoExtractor):
|
||||
IE_NAME = 'croatian.film'
|
||||
_VALID_URL = r'https://?(?:www\.)?croatian\.film/[a-z]{2}/[^/?#]+/(?P<id>\d+)'
|
||||
_GEO_COUNTRIES = ['HR']
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://www.croatian.film/hr/films/72472',
|
||||
'info_dict': {
|
||||
'id': '1078340774',
|
||||
'ext': 'mp4',
|
||||
'title': '“ŠKAFETIN”, r. Paško Vukasović',
|
||||
'uploader': 'croatian.film',
|
||||
'uploader_id': 'user94192658',
|
||||
'uploader_url': 'https://vimeo.com/user94192658',
|
||||
'duration': 1357,
|
||||
'thumbnail': 'https://i.vimeocdn.com/video/2008556407-40eb1315ec11be5fcb8dda4d7059675b0881e182b9fc730892e267db72cb57f5-d',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
'expected_warnings': ['Failed to parse XML: not well-formed'],
|
||||
}, {
|
||||
# geo-restricted but works with xff
|
||||
'url': 'https://www.croatian.film/en/films/77144',
|
||||
'info_dict': {
|
||||
'id': '1144997795',
|
||||
'ext': 'mp4',
|
||||
'title': '“ROKO” r. Ivana Marinić Kragić',
|
||||
'uploader': 'croatian.film',
|
||||
'uploader_id': 'user94192658',
|
||||
'uploader_url': 'https://vimeo.com/user94192658',
|
||||
'duration': 1023,
|
||||
'thumbnail': 'https://i.vimeocdn.com/video/2093793231-11c2928698ff8347489e679b4d563a576e7acd0681ce95b383a9a25f6adb5e8f-d',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
'expected_warnings': ['Failed to parse XML: not well-formed'],
|
||||
}, {
|
||||
'url': 'https://www.croatian.film/en/films/75904/watch',
|
||||
'info_dict': {
|
||||
'id': '1134883757',
|
||||
'ext': 'mp4',
|
||||
'title': '"CARPE DIEM" r. Nina Damjanović',
|
||||
'uploader': 'croatian.film',
|
||||
'uploader_id': 'user94192658',
|
||||
'uploader_url': 'https://vimeo.com/user94192658',
|
||||
'duration': 1123,
|
||||
'thumbnail': 'https://i.vimeocdn.com/video/2080022187-bb691c470c28c4d979258cf235e594bf9a11c14b837a0784326c25c95edd83f9-d',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
'expected_warnings': ['Failed to parse XML: not well-formed'],
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
api_data = self._download_json(
|
||||
f'https://api.croatian.film/api/videos/{display_id}',
|
||||
display_id)
|
||||
|
||||
if errors := traverse_obj(api_data, ('errors', lambda _, v: v['code'])):
|
||||
codes = traverse_obj(errors, (..., 'code', {str}))
|
||||
if 'INVALID_COUNTRY' in codes:
|
||||
self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
|
||||
raise ExtractorError(join_nonempty(
|
||||
*(traverse_obj(errors, (..., 'details', {str})) or codes),
|
||||
delim='; '))
|
||||
|
||||
vimeo_id = self._search_regex(
|
||||
r'/videos/(\d+)', api_data['video']['vimeoURL'], 'vimeo ID')
|
||||
|
||||
return self.url_result(
|
||||
VimeoIE._smuggle_referrer(f'https://player.vimeo.com/video/{vimeo_id}', url),
|
||||
VimeoIE, vimeo_id)
|
||||
@@ -1,5 +1,6 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import int_or_none
|
||||
from ..utils import int_or_none, url_or_none
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class DigitekaIE(InfoExtractor):
|
||||
@@ -25,74 +26,56 @@ class DigitekaIE(InfoExtractor):
|
||||
)/(?P<id>[\d+a-z]+)'''
|
||||
_EMBED_REGEX = [r'<(?:iframe|script)[^>]+src=["\'](?P<url>(?:https?:)?//(?:www\.)?ultimedia\.com/deliver/(?:generic|musique)(?:/[^/]+)*/(?:src|article)/[\d+a-z]+)']
|
||||
_TESTS = [{
|
||||
# news
|
||||
'url': 'https://www.ultimedia.com/default/index/videogeneric/id/s8uk0r',
|
||||
'md5': '276a0e49de58c7e85d32b057837952a2',
|
||||
'url': 'https://www.ultimedia.com/default/index/videogeneric/id/3x5x55k',
|
||||
'info_dict': {
|
||||
'id': 's8uk0r',
|
||||
'id': '3x5x55k',
|
||||
'ext': 'mp4',
|
||||
'title': 'Loi sur la fin de vie: le texte prévoit un renforcement des directives anticipées',
|
||||
'title': 'Il est passionné de DS',
|
||||
'thumbnail': r're:^https?://.*\.jpg',
|
||||
'duration': 74,
|
||||
'upload_date': '20150317',
|
||||
'timestamp': 1426604939,
|
||||
'uploader_id': '3fszv',
|
||||
'duration': 89,
|
||||
'upload_date': '20251012',
|
||||
'timestamp': 1760285363,
|
||||
'uploader_id': '3pz33',
|
||||
},
|
||||
}, {
|
||||
# music
|
||||
'url': 'https://www.ultimedia.com/default/index/videomusic/id/xvpfp8',
|
||||
'md5': '2ea3513813cf230605c7e2ffe7eca61c',
|
||||
'info_dict': {
|
||||
'id': 'xvpfp8',
|
||||
'ext': 'mp4',
|
||||
'title': 'Two - C\'est La Vie (clip)',
|
||||
'thumbnail': r're:^https?://.*\.jpg',
|
||||
'duration': 233,
|
||||
'upload_date': '20150224',
|
||||
'timestamp': 1424760500,
|
||||
'uploader_id': '3rfzk',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.digiteka.net/deliver/generic/iframe/mdtk/01637594/src/lqm3kl/zone/1/showtitle/1/autoplay/yes',
|
||||
'only_matching': True,
|
||||
'params': {'skip_download': True},
|
||||
}]
|
||||
_IFRAME_MD_ID = '01836272' # One static ID working for Ultimedia iframes
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = self._match_valid_url(url)
|
||||
video_id = mobj.group('id')
|
||||
video_type = mobj.group('embed_type') or mobj.group('site_type')
|
||||
if video_type == 'music':
|
||||
video_type = 'musique'
|
||||
video_id = self._match_id(url)
|
||||
|
||||
deliver_info = self._download_json(
|
||||
f'http://www.ultimedia.com/deliver/video?video={video_id}&topic={video_type}',
|
||||
video_id)
|
||||
|
||||
yt_id = deliver_info.get('yt_id')
|
||||
if yt_id:
|
||||
return self.url_result(yt_id, 'Youtube')
|
||||
|
||||
jwconf = deliver_info['jwconf']
|
||||
video_info = self._download_json(
|
||||
f'https://www.ultimedia.com/player/getConf/{self._IFRAME_MD_ID}/1/{video_id}', video_id,
|
||||
note='Downloading player configuration')['video']
|
||||
|
||||
formats = []
|
||||
for source in jwconf['playlist'][0]['sources']:
|
||||
formats.append({
|
||||
'url': source['file'],
|
||||
'format_id': source.get('label'),
|
||||
})
|
||||
subtitles = {}
|
||||
|
||||
title = deliver_info['title']
|
||||
thumbnail = jwconf.get('image')
|
||||
duration = int_or_none(deliver_info.get('duration'))
|
||||
timestamp = int_or_none(deliver_info.get('release_time'))
|
||||
uploader_id = deliver_info.get('owner_id')
|
||||
if hls_url := traverse_obj(video_info, ('media_sources', 'hls', 'hls_auto', {url_or_none})):
|
||||
fmts, subs = self._extract_m3u8_formats_and_subtitles(
|
||||
hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
|
||||
formats.extend(fmts)
|
||||
self._merge_subtitles(subs, target=subtitles)
|
||||
|
||||
for format_id, mp4_url in traverse_obj(video_info, ('media_sources', 'mp4', {dict.items}, ...)):
|
||||
if not mp4_url:
|
||||
continue
|
||||
formats.append({
|
||||
'url': mp4_url,
|
||||
'format_id': format_id,
|
||||
'height': int_or_none(format_id.partition('_')[2]),
|
||||
'ext': 'mp4',
|
||||
})
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'thumbnail': thumbnail,
|
||||
'duration': duration,
|
||||
'timestamp': timestamp,
|
||||
'uploader_id': uploader_id,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
**traverse_obj(video_info, {
|
||||
'title': ('title', {str}),
|
||||
'thumbnail': ('image', {url_or_none}),
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'timestamp': ('creationDate', {int_or_none}),
|
||||
'uploader_id': ('ownerId', {str}),
|
||||
}),
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ from ..utils import (
|
||||
try_get,
|
||||
unified_timestamp,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class DPlayBaseIE(InfoExtractor):
|
||||
@@ -1053,7 +1054,7 @@ class DiscoveryPlusIndiaIE(DiscoveryPlusBaseIE):
|
||||
|
||||
|
||||
class DiscoveryNetworksDeIE(DiscoveryPlusBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:tlc|dmax)\.de|dplay\.co\.uk)/(?:programme|show|sendungen)/(?P<programme>[^/]+)/(?:video/)?(?P<alternate_id>[^/]+)'
|
||||
_VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:tlc|dmax)\.de)/(?:programme|show|sendungen)/(?P<programme>[^/?#]+)/(?:video/)?(?P<alternate_id>[^/?#]+)'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://dmax.de/sendungen/goldrausch-in-australien/german-gold',
|
||||
@@ -1074,6 +1075,7 @@ class DiscoveryNetworksDeIE(DiscoveryPlusBaseIE):
|
||||
'creators': ['DMAX'],
|
||||
'thumbnail': 'https://eu1-prod-images.disco-api.com/2023/05/09/f72fb510-7992-3b12-af7f-f16a2c22d1e3.jpeg',
|
||||
'tags': ['schatzsucher', 'schatz', 'nugget', 'bodenschätze', 'down under', 'australien', 'goldrausch'],
|
||||
'categories': ['Gold', 'Schatzsucher'],
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}, {
|
||||
@@ -1100,20 +1102,96 @@ class DiscoveryNetworksDeIE(DiscoveryPlusBaseIE):
|
||||
}, {
|
||||
'url': 'https://www.dmax.de/programme/dmax-highlights/video/tuning-star-sidney-hoffmann-exklusiv-bei-dmax/191023082312316',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://www.dplay.co.uk/show/ghost-adventures/video/hotel-leger-103620/EHD_280313B',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://tlc.de/sendungen/breaking-amish/die-welt-da-drauen/',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://dmax.de/sendungen/feuerwache-3-alarm-in-muenchen/24-stunden-auf-der-feuerwache-3',
|
||||
'info_dict': {
|
||||
'id': '8873549',
|
||||
'ext': 'mp4',
|
||||
'title': '24 Stunden auf der Feuerwache 3',
|
||||
'description': 'md5:f3084ef6170bfb79f9a6e0c030e09330',
|
||||
'display_id': 'feuerwache-3-alarm-in-muenchen/24-stunden-auf-der-feuerwache-3',
|
||||
'episode': 'Episode 1',
|
||||
'episode_number': 1,
|
||||
'season': 'Season 1',
|
||||
'season_number': 1,
|
||||
'series': 'Feuerwache 3 - Alarm in München',
|
||||
'duration': 2632.0,
|
||||
'upload_date': '20251016',
|
||||
'timestamp': 1760645100,
|
||||
'creators': ['DMAX'],
|
||||
'thumbnail': 'https://eu1-prod-images.disco-api.com/2025/10/14/0bdee68c-a8d8-33d9-9204-16eb61108552.jpeg',
|
||||
'tags': [],
|
||||
'categories': ['DMAX Originals', 'Jobs', 'Blaulicht'],
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}, {
|
||||
'url': 'https://tlc.de/sendungen/ghost-adventures/der-poltergeist-im-kostumladen',
|
||||
'info_dict': {
|
||||
'id': '4550602',
|
||||
'ext': 'mp4',
|
||||
'title': 'Der Poltergeist im Kostümladen',
|
||||
'description': 'md5:20b52b9736a0a3a7873d19a238fad7fc',
|
||||
'display_id': 'ghost-adventures/der-poltergeist-im-kostumladen',
|
||||
'episode': 'Episode 1',
|
||||
'episode_number': 1,
|
||||
'season': 'Season 25',
|
||||
'season_number': 25,
|
||||
'series': 'Ghost Adventures',
|
||||
'duration': 2493.0,
|
||||
'upload_date': '20241223',
|
||||
'timestamp': 1734948900,
|
||||
'creators': ['TLC'],
|
||||
'thumbnail': 'https://eu1-prod-images.disco-api.com/2023/04/05/59941d26-a81b-365f-829f-69d8cd81fd0f.jpeg',
|
||||
'tags': [],
|
||||
'categories': ['Paranormal', 'Gruselig!'],
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}, {
|
||||
'url': 'https://tlc.de/sendungen/evil-gesichter-des-boesen/das-geheimnis-meines-bruders',
|
||||
'info_dict': {
|
||||
'id': '7792288',
|
||||
'ext': 'mp4',
|
||||
'title': 'Das Geheimnis meines Bruders',
|
||||
'description': 'md5:3167550bb582eb9c92875c86a0a20882',
|
||||
'display_id': 'evil-gesichter-des-boesen/das-geheimnis-meines-bruders',
|
||||
'episode': 'Episode 1',
|
||||
'episode_number': 1,
|
||||
'season': 'Season 1',
|
||||
'season_number': 1,
|
||||
'series': 'Evil - Gesichter des Bösen',
|
||||
'duration': 2626.0,
|
||||
'upload_date': '20240926',
|
||||
'timestamp': 1727388000,
|
||||
'creators': ['TLC'],
|
||||
'thumbnail': 'https://eu1-prod-images.disco-api.com/2024/11/29/e9f3e3ae-74ec-3631-81b7-fc7bbe844741.jpeg',
|
||||
'tags': 'count:13',
|
||||
'categories': ['True Crime', 'Mord'],
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
domain, programme, alternate_id = self._match_valid_url(url).groups()
|
||||
country = 'GB' if domain == 'dplay.co.uk' else 'DE'
|
||||
realm = 'questuk' if country == 'GB' else domain.replace('.', '')
|
||||
return self._get_disco_api_info(
|
||||
url, f'{programme}/{alternate_id}', 'eu1-prod.disco-api.com', realm, country)
|
||||
display_id = f'{programme}/{alternate_id}'
|
||||
meta = self._download_json(
|
||||
f'https://de-api.loma-cms.com/feloma/videos/{alternate_id}/',
|
||||
display_id, query={
|
||||
'environment': domain.split('.')[0],
|
||||
'v': '2',
|
||||
'filter[show.slug]': programme,
|
||||
}, fatal=False)
|
||||
video_id = traverse_obj(meta, ('uid', {str}, {lambda s: s[-7:]})) or display_id
|
||||
|
||||
disco_api_info = self._get_disco_api_info(
|
||||
url, video_id, 'eu1-prod.disco-api.com', domain.replace('.', ''), 'DE')
|
||||
disco_api_info['display_id'] = display_id
|
||||
disco_api_info['categories'] = traverse_obj(meta, (
|
||||
'taxonomies', lambda _, v: v['category'] == 'genre', 'title', {str.strip}, filter, all, filter))
|
||||
|
||||
return disco_api_info
|
||||
|
||||
def _update_disco_api_headers(self, headers, disco_base, display_id, realm):
|
||||
headers.update({
|
||||
|
||||
@@ -14,7 +14,7 @@ from ..utils import (
|
||||
|
||||
|
||||
class DropboxIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?dropbox\.com/(?:(?:e/)?scl/fi|sh?)/(?P<id>\w+)'
|
||||
_VALID_URL = r'https?://(?:www\.)?dropbox\.com/(?:(?:e/)?scl/f[io]|sh?)/(?P<id>\w+)'
|
||||
_TESTS = [
|
||||
{
|
||||
'url': 'https://www.dropbox.com/s/nelirfsxnmcfbfh/youtube-dl%20test%20video%20%27%C3%A4%22BaW_jenozKc.mp4?dl=0',
|
||||
@@ -35,6 +35,9 @@ class DropboxIE(InfoExtractor):
|
||||
}, {
|
||||
'url': 'https://www.dropbox.com/e/scl/fi/r2kd2skcy5ylbbta5y1pz/DJI_0003.MP4?dl=0&rlkey=wcdgqangn7t3lnmmv6li9mu9h',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://www.dropbox.com/scl/fo/zjfqse5txqfd7twa8iewj/AOfZzSYWUSKle2HD7XF7kzQ/A-BEAT%20C.mp4?rlkey=6tg3jkp4tv6a5vt58a6dag0mm&dl=0',
|
||||
'only_matching': True,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@@ -4,8 +4,6 @@ import urllib.parse
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..compat import compat_etree_fromstring
|
||||
from ..networking import Request
|
||||
from ..networking.exceptions import network_exceptions
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
clean_html,
|
||||
@@ -64,9 +62,6 @@ class FacebookIE(InfoExtractor):
|
||||
class=(?P<q1>[\'"])[^\'"]*\bfb-(?:video|post)\b[^\'"]*(?P=q1)[^>]+
|
||||
data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2)''',
|
||||
]
|
||||
_LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1'
|
||||
_CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
|
||||
_NETRC_MACHINE = 'facebook'
|
||||
IE_NAME = 'facebook'
|
||||
|
||||
_VIDEO_PAGE_TEMPLATE = 'https://www.facebook.com/video/video.php?v=%s'
|
||||
@@ -469,65 +464,6 @@ class FacebookIE(InfoExtractor):
|
||||
'graphURI': '/api/graphql/',
|
||||
}
|
||||
|
||||
def _perform_login(self, username, password):
|
||||
login_page_req = Request(self._LOGIN_URL)
|
||||
self._set_cookie('facebook.com', 'locale', 'en_US')
|
||||
login_page = self._download_webpage(login_page_req, None,
|
||||
note='Downloading login page',
|
||||
errnote='Unable to download login page')
|
||||
lsd = self._search_regex(
|
||||
r'<input type="hidden" name="lsd" value="([^"]*)"',
|
||||
login_page, 'lsd')
|
||||
lgnrnd = self._search_regex(r'name="lgnrnd" value="([^"]*?)"', login_page, 'lgnrnd')
|
||||
|
||||
login_form = {
|
||||
'email': username,
|
||||
'pass': password,
|
||||
'lsd': lsd,
|
||||
'lgnrnd': lgnrnd,
|
||||
'next': 'http://facebook.com/home.php',
|
||||
'default_persistent': '0',
|
||||
'legacy_return': '1',
|
||||
'timezone': '-60',
|
||||
'trynum': '1',
|
||||
}
|
||||
request = Request(self._LOGIN_URL, urlencode_postdata(login_form))
|
||||
request.headers['Content-Type'] = 'application/x-www-form-urlencoded'
|
||||
try:
|
||||
login_results = self._download_webpage(request, None,
|
||||
note='Logging in', errnote='unable to fetch login page')
|
||||
if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
|
||||
error = self._html_search_regex(
|
||||
r'(?s)<div[^>]+class=(["\']).*?login_error_box.*?\1[^>]*><div[^>]*>.*?</div><div[^>]*>(?P<error>.+?)</div>',
|
||||
login_results, 'login error', default=None, group='error')
|
||||
if error:
|
||||
raise ExtractorError(f'Unable to login: {error}', expected=True)
|
||||
self.report_warning('unable to log in: bad username/password, or exceeded login rate limit (~3/min). Check credentials or wait.')
|
||||
return
|
||||
|
||||
fb_dtsg = self._search_regex(
|
||||
r'name="fb_dtsg" value="(.+?)"', login_results, 'fb_dtsg', default=None)
|
||||
h = self._search_regex(
|
||||
r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h', default=None)
|
||||
|
||||
if not fb_dtsg or not h:
|
||||
return
|
||||
|
||||
check_form = {
|
||||
'fb_dtsg': fb_dtsg,
|
||||
'h': h,
|
||||
'name_action_selected': 'dont_save',
|
||||
}
|
||||
check_req = Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
|
||||
check_req.headers['Content-Type'] = 'application/x-www-form-urlencoded'
|
||||
check_response = self._download_webpage(check_req, None,
|
||||
note='Confirming login')
|
||||
if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
|
||||
self.report_warning('Unable to confirm login, you have to login in your browser and authorize the login.')
|
||||
except network_exceptions as err:
|
||||
self.report_warning(f'unable to log in: {err}')
|
||||
return
|
||||
|
||||
def _extract_from_url(self, url, video_id):
|
||||
webpage = self._download_webpage(
|
||||
url.replace('://m.facebook.com/', '://www.facebook.com/'), video_id)
|
||||
|
||||
@@ -5,6 +5,7 @@ from .common import InfoExtractor
|
||||
from ..networking import Request
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
UserNotLive,
|
||||
js_to_json,
|
||||
traverse_obj,
|
||||
update_url_query,
|
||||
@@ -205,6 +206,9 @@ class FC2LiveIE(InfoExtractor):
|
||||
'client_app': 'browser_hls',
|
||||
'ipv6': '',
|
||||
}), headers={'X-Requested-With': 'XMLHttpRequest'})
|
||||
# A non-zero 'status' indicates the stream is not live, so check truthiness
|
||||
if traverse_obj(control_server, ('status', {int})) and 'control_token' not in control_server:
|
||||
raise UserNotLive(video_id=video_id)
|
||||
self._set_cookie('live.fc2.com', 'l_ortkn', control_server['orz_raw'])
|
||||
|
||||
ws_url = update_url_query(control_server['url'], {'control_token': control_server['control_token']})
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import clean_html
|
||||
from ..utils.traversal import (
|
||||
find_element,
|
||||
find_elements,
|
||||
traverse_obj,
|
||||
)
|
||||
|
||||
|
||||
class FilmArchivIE(InfoExtractor):
|
||||
IE_DESC = 'FILMARCHIV ON'
|
||||
_VALID_URL = r'https?://(?:www\.)?filmarchiv\.at/de/filmarchiv-on/video/(?P<id>f_[0-9a-zA-Z]{5,})'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.filmarchiv.at/de/filmarchiv-on/video/f_0305p7xKrXUPBwoNE9x6mh',
|
||||
'md5': '54a6596f6a84624531866008a77fa27a',
|
||||
'info_dict': {
|
||||
'id': 'f_0305p7xKrXUPBwoNE9x6mh',
|
||||
'ext': 'mp4',
|
||||
'title': 'Der Wurstelprater zur Kaiserzeit',
|
||||
'description': 'md5:9843f92df5cc9a4975cee7aabcf6e3b2',
|
||||
'thumbnail': r're:https://cdn\.filmarchiv\.at/f_0305/p7xKrXUPBwoNE9x6mh_v1/poster\.jpg',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.filmarchiv.at/de/filmarchiv-on/video/f_0306vI3wO0tJIsfrqYFQXF',
|
||||
'md5': '595385d7f54cb6529140ee8de7d1c3c7',
|
||||
'info_dict': {
|
||||
'id': 'f_0306vI3wO0tJIsfrqYFQXF',
|
||||
'ext': 'mp4',
|
||||
'title': 'Vor 70 Jahren: Wettgehen der Briefträger in Wien',
|
||||
'description': 'md5:b2a2e4230923cd1969d471c552e62811',
|
||||
'thumbnail': r're:https://cdn\.filmarchiv\.at/f_0306/vI3wO0tJIsfrqYFQXF_v1/poster\.jpg',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
media_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, media_id)
|
||||
path = '/'.join((media_id[:6], media_id[6:]))
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||
f'https://cdn.filmarchiv.at/{path}_v1_sv1/playlist.m3u8', media_id)
|
||||
|
||||
return {
|
||||
'id': media_id,
|
||||
'title': traverse_obj(webpage, ({find_element(tag='title-div')}, {clean_html})),
|
||||
'description': traverse_obj(webpage, (
|
||||
{find_elements(tag='div', attr='class', value=r'.*\bborder-base-content\b', regex=True)}, ...,
|
||||
{find_elements(tag='div', attr='class', value=r'.*\bprose\b', html=False, regex=True)}, ...,
|
||||
{clean_html}, any)),
|
||||
'thumbnail': f'https://cdn.filmarchiv.at/{path}_v1/poster.jpg',
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
}
|
||||
@@ -10,7 +10,7 @@ from ..utils import (
|
||||
unified_strdate,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
from ..utils.traversal import require, traverse_obj
|
||||
|
||||
|
||||
class FirstTVIE(InfoExtractor):
|
||||
@@ -129,3 +129,36 @@ class FirstTVIE(InfoExtractor):
|
||||
return self.playlist_result(
|
||||
self._entries(items), display_id, self._og_search_title(webpage, default=None),
|
||||
thumbnail=self._og_search_thumbnail(webpage, default=None))
|
||||
|
||||
|
||||
class FirstTVLiveIE(InfoExtractor):
|
||||
IE_NAME = '1tv:live'
|
||||
IE_DESC = 'Первый канал (прямой эфир)'
|
||||
_VALID_URL = r'https?://(?:www\.)?1tv\.ru/live'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://www.1tv.ru/live',
|
||||
'info_dict': {
|
||||
'id': 'live',
|
||||
'ext': 'mp4',
|
||||
'title': r're:ПЕРВЫЙ КАНАЛ ПРЯМОЙ ЭФИР СМОТРЕТЬ ОНЛАЙН \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
|
||||
'live_status': 'is_live',
|
||||
},
|
||||
'params': {'skip_download': 'livestream'},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = 'live'
|
||||
webpage = self._download_webpage(url, display_id, fatal=False)
|
||||
|
||||
streams_list = self._download_json('https://stream.1tv.ru/api/playlist/1tvch-v1_as_array.json', display_id)
|
||||
mpd_url = traverse_obj(streams_list, ('mpd', ..., {url_or_none}, any, {require('mpd url')}))
|
||||
# FFmpeg needs to be passed -re to not seek past live window. This is handled by core
|
||||
formats, _ = self._extract_mpd_formats_and_subtitles(mpd_url, display_id, mpd_id='dash')
|
||||
|
||||
return {
|
||||
'id': display_id,
|
||||
'title': self._html_extract_title(webpage),
|
||||
'formats': formats,
|
||||
'is_live': True,
|
||||
}
|
||||
|
||||
@@ -6,15 +6,15 @@ from ..utils import (
|
||||
OnDemandPagedList,
|
||||
clean_html,
|
||||
determine_ext,
|
||||
float_or_none,
|
||||
format_field,
|
||||
int_or_none,
|
||||
join_nonempty,
|
||||
parse_codecs,
|
||||
parse_iso8601,
|
||||
url_or_none,
|
||||
urljoin,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
from ..utils.traversal import require, traverse_obj
|
||||
|
||||
|
||||
class FloatplaneBaseIE(InfoExtractor):
|
||||
@@ -50,37 +50,31 @@ class FloatplaneBaseIE(InfoExtractor):
|
||||
media_id = media['id']
|
||||
media_typ = media.get('type') or 'video'
|
||||
|
||||
metadata = self._download_json(
|
||||
f'{self._BASE_URL}/api/v3/content/{media_typ}', media_id, query={'id': media_id},
|
||||
note=f'Downloading {media_typ} metadata', impersonate=self._IMPERSONATE_TARGET)
|
||||
|
||||
stream = self._download_json(
|
||||
f'{self._BASE_URL}/api/v2/cdn/delivery', media_id, query={
|
||||
'type': 'vod' if media_typ == 'video' else 'aod',
|
||||
'guid': metadata['guid'],
|
||||
}, note=f'Downloading {media_typ} stream data',
|
||||
f'{self._BASE_URL}/api/v3/delivery/info', media_id,
|
||||
query={'scenario': 'onDemand', 'entityId': media_id},
|
||||
note=f'Downloading {media_typ} stream data',
|
||||
impersonate=self._IMPERSONATE_TARGET)
|
||||
|
||||
path_template = traverse_obj(stream, ('resource', 'uri', {str}))
|
||||
metadata = self._download_json(
|
||||
f'{self._BASE_URL}/api/v3/content/{media_typ}', media_id,
|
||||
f'Downloading {media_typ} metadata', query={'id': media_id},
|
||||
fatal=False, impersonate=self._IMPERSONATE_TARGET)
|
||||
|
||||
def format_path(params):
|
||||
path = path_template
|
||||
for i, val in (params or {}).items():
|
||||
path = path.replace(f'{{qualityLevelParams.{i}}}', val)
|
||||
return path
|
||||
cdn_base_url = traverse_obj(stream, (
|
||||
'groups', 0, 'origins', ..., 'url', {url_or_none}, any, {require('cdn base url')}))
|
||||
|
||||
formats = []
|
||||
for quality in traverse_obj(stream, ('resource', 'data', 'qualityLevels', ...)):
|
||||
url = urljoin(stream['cdn'], format_path(traverse_obj(
|
||||
stream, ('resource', 'data', 'qualityLevelParams', quality['name'], {dict}))))
|
||||
format_id = traverse_obj(quality, ('name', {str}))
|
||||
for variant in traverse_obj(stream, ('groups', 0, 'variants', lambda _, v: v['url'])):
|
||||
format_url = urljoin(cdn_base_url, variant['url'])
|
||||
format_id = traverse_obj(variant, ('name', {str}))
|
||||
hls_aes = {}
|
||||
m3u8_data = None
|
||||
|
||||
# If we need impersonation for the API, then we need it for HLS keys too: extract in advance
|
||||
if self._IMPERSONATE_TARGET is not None:
|
||||
m3u8_data = self._download_webpage(
|
||||
url, media_id, fatal=False, impersonate=self._IMPERSONATE_TARGET, headers=self._HEADERS,
|
||||
format_url, media_id, fatal=False, impersonate=self._IMPERSONATE_TARGET, headers=self._HEADERS,
|
||||
note=join_nonempty('Downloading', format_id, 'm3u8 information', delim=' '),
|
||||
errnote=join_nonempty('Failed to download', format_id, 'm3u8 information', delim=' '))
|
||||
if not m3u8_data:
|
||||
@@ -98,18 +92,34 @@ class FloatplaneBaseIE(InfoExtractor):
|
||||
hls_aes['key'] = urlh.read().hex()
|
||||
|
||||
formats.append({
|
||||
**traverse_obj(quality, {
|
||||
**traverse_obj(variant, {
|
||||
'format_note': ('label', {str}),
|
||||
'width': ('width', {int}),
|
||||
'height': ('height', {int}),
|
||||
'width': ('meta', 'video', 'width', {int_or_none}),
|
||||
'height': ('meta', 'video', 'height', {int_or_none}),
|
||||
'vcodec': ('meta', 'video', 'codec', {str}),
|
||||
'acodec': ('meta', 'audio', 'codec', {str}),
|
||||
'vbr': ('meta', 'video', 'bitrate', 'average', {int_or_none(scale=1000)}),
|
||||
'abr': ('meta', 'audio', 'bitrate', 'average', {int_or_none(scale=1000)}),
|
||||
'audio_channels': ('meta', 'audio', 'channelCount', {int_or_none}),
|
||||
'fps': ('meta', 'video', 'fps', {float_or_none}),
|
||||
}),
|
||||
**parse_codecs(quality.get('codecs')),
|
||||
'url': url,
|
||||
'ext': determine_ext(url.partition('/chunk.m3u8')[0], 'mp4'),
|
||||
'url': format_url,
|
||||
'ext': determine_ext(format_url.partition('/chunk.m3u8')[0], 'mp4'),
|
||||
'format_id': format_id,
|
||||
'hls_media_playlist_data': m3u8_data,
|
||||
'hls_aes': hls_aes or None,
|
||||
})
|
||||
|
||||
subtitles = {}
|
||||
automatic_captions = {}
|
||||
for sub_data in traverse_obj(metadata, ('textTracks', lambda _, v: url_or_none(v['src']))):
|
||||
sub_lang = sub_data.get('language') or 'en'
|
||||
sub_entry = {'url': sub_data['src']}
|
||||
if sub_data.get('generated'):
|
||||
automatic_captions.setdefault(sub_lang, []).append(sub_entry)
|
||||
else:
|
||||
subtitles.setdefault(sub_lang, []).append(sub_entry)
|
||||
|
||||
items.append({
|
||||
**common_info,
|
||||
'id': media_id,
|
||||
@@ -119,6 +129,8 @@ class FloatplaneBaseIE(InfoExtractor):
|
||||
'thumbnail': ('thumbnail', 'path', {url_or_none}),
|
||||
}),
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'automatic_captions': automatic_captions,
|
||||
})
|
||||
|
||||
post_info = {
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import int_or_none, parse_iso8601, url_or_none
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class FrontoBaseIE(InfoExtractor):
|
||||
def _get_auth_headers(self, url):
|
||||
return traverse_obj(self._get_cookies(url), {
|
||||
'authorization': ('frAccessToken', 'value', {lambda token: f'Bearer {token}' if token else None}),
|
||||
})
|
||||
|
||||
|
||||
class FrontroVideoBaseIE(FrontoBaseIE):
|
||||
_CHANNEL_ID = None
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
metadata = self._download_json(
|
||||
'https://api.frontrow.cc/query', video_id, data=json.dumps({
|
||||
'operationName': 'Video',
|
||||
'variables': {'channelID': self._CHANNEL_ID, 'videoID': video_id},
|
||||
'query': '''query Video($channelID: ID!, $videoID: ID!) {
|
||||
video(ChannelID: $channelID, VideoID: $videoID) {
|
||||
... on Video {title description updatedAt thumbnail createdAt duration likeCount comments views url hasAccess}
|
||||
}
|
||||
}''',
|
||||
}).encode(), headers={
|
||||
'content-type': 'application/json',
|
||||
**self._get_auth_headers(url),
|
||||
})['data']['video']
|
||||
if not traverse_obj(metadata, 'hasAccess'):
|
||||
self.raise_login_required()
|
||||
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(metadata['url'], video_id)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
**traverse_obj(metadata, {
|
||||
'title': ('title', {str}),
|
||||
'description': ('description', {str}),
|
||||
'thumbnail': ('thumbnail', {url_or_none}),
|
||||
'timestamp': ('createdAt', {parse_iso8601}),
|
||||
'modified_timestamp': ('updatedAt', {parse_iso8601}),
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'like_count': ('likeCount', {int_or_none}),
|
||||
'comment_count': ('comments', {int_or_none}),
|
||||
'view_count': ('views', {int_or_none}),
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
class FrontroGroupBaseIE(FrontoBaseIE):
|
||||
_CHANNEL_ID = None
|
||||
_VIDEO_EXTRACTOR = None
|
||||
_VIDEO_URL_TMPL = None
|
||||
|
||||
def _real_extract(self, url):
|
||||
group_id = self._match_id(url)
|
||||
|
||||
metadata = self._download_json(
|
||||
'https://api.frontrow.cc/query', group_id, note='Downloading playlist metadata',
|
||||
data=json.dumps({
|
||||
'operationName': 'PaginatedStaticPageContainer',
|
||||
'variables': {'channelID': self._CHANNEL_ID, 'first': 500, 'pageContainerID': group_id},
|
||||
'query': '''query PaginatedStaticPageContainer($channelID: ID!, $pageContainerID: ID!) {
|
||||
pageContainer(ChannelID: $channelID, PageContainerID: $pageContainerID) {
|
||||
... on StaticPageContainer { id title updatedAt createdAt itemRefs {edges {node {
|
||||
id contentItem { ... on ItemVideo { videoItem: item {
|
||||
id
|
||||
}}}
|
||||
}}}
|
||||
}
|
||||
}
|
||||
}''',
|
||||
}).encode(), headers={
|
||||
'content-type': 'application/json',
|
||||
**self._get_auth_headers(url),
|
||||
})['data']['pageContainer']
|
||||
|
||||
entries = []
|
||||
for video_id in traverse_obj(metadata, (
|
||||
'itemRefs', 'edges', ..., 'node', 'contentItem', 'videoItem', 'id', {str}),
|
||||
):
|
||||
entries.append(self.url_result(
|
||||
self._VIDEO_URL_TMPL % video_id, self._VIDEO_EXTRACTOR, video_id))
|
||||
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
'id': group_id,
|
||||
'entries': entries,
|
||||
**traverse_obj(metadata, {
|
||||
'title': ('title', {str}),
|
||||
'timestamp': ('createdAt', {parse_iso8601}),
|
||||
'modified_timestamp': ('updatedAt', {parse_iso8601}),
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
class TheChosenIE(FrontroVideoBaseIE):
|
||||
_CHANNEL_ID = '12884901895'
|
||||
|
||||
_VALID_URL = r'https?://(?:www\.)?watch\.thechosen\.tv/video/(?P<id>[0-9]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://watch.thechosen.tv/video/184683594325',
|
||||
'md5': '3f878b689588c71b38ec9943c54ff5b0',
|
||||
'info_dict': {
|
||||
'id': '184683594325',
|
||||
'ext': 'mp4',
|
||||
'title': 'Season 3 Episode 2: Two by Two',
|
||||
'description': 'md5:174c373756ecc8df46b403f4fcfbaf8c',
|
||||
'comment_count': int,
|
||||
'view_count': int,
|
||||
'like_count': int,
|
||||
'duration': 4212,
|
||||
'thumbnail': r're:https://fastly\.frontrowcdn\.com/channels/12884901895/VIDEO_THUMBNAIL/184683594325/',
|
||||
'timestamp': 1698954546,
|
||||
'upload_date': '20231102',
|
||||
'modified_timestamp': int,
|
||||
'modified_date': str,
|
||||
},
|
||||
}, {
|
||||
'url': 'https://watch.thechosen.tv/video/184683596189',
|
||||
'md5': 'd581562f9d29ce82f5b7770415334151',
|
||||
'info_dict': {
|
||||
'id': '184683596189',
|
||||
'ext': 'mp4',
|
||||
'title': 'Season 4 Episode 8: Humble',
|
||||
'description': 'md5:20a57bead43da1cf77cd5b0fe29bbc76',
|
||||
'comment_count': int,
|
||||
'view_count': int,
|
||||
'like_count': int,
|
||||
'duration': 5092,
|
||||
'thumbnail': r're:https://fastly\.frontrowcdn\.com/channels/12884901895/VIDEO_THUMBNAIL/184683596189/',
|
||||
'timestamp': 1715019474,
|
||||
'upload_date': '20240506',
|
||||
'modified_timestamp': int,
|
||||
'modified_date': str,
|
||||
},
|
||||
}]
|
||||
|
||||
|
||||
class TheChosenGroupIE(FrontroGroupBaseIE):
|
||||
_CHANNEL_ID = '12884901895'
|
||||
_VIDEO_EXTRACTOR = TheChosenIE
|
||||
_VIDEO_URL_TMPL = 'https://watch.thechosen.tv/video/%s'
|
||||
|
||||
_VALID_URL = r'https?://(?:www\.)?watch\.thechosen\.tv/group/(?P<id>[0-9]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://watch.thechosen.tv/group/309237658592',
|
||||
'info_dict': {
|
||||
'id': '309237658592',
|
||||
'title': 'Season 3',
|
||||
'timestamp': 1746203969,
|
||||
'upload_date': '20250502',
|
||||
'modified_timestamp': int,
|
||||
'modified_date': str,
|
||||
},
|
||||
'playlist_count': 8,
|
||||
}]
|
||||
@@ -821,13 +821,17 @@ class GenericIE(InfoExtractor):
|
||||
'Referer': smuggled_data.get('referer'),
|
||||
}), impersonate=impersonate)
|
||||
except ExtractorError as e:
|
||||
if not (isinstance(e.cause, HTTPError) and e.cause.status == 403
|
||||
and e.cause.response.get_header('cf-mitigated') == 'challenge'
|
||||
and e.cause.response.extensions.get('impersonate') is None):
|
||||
if not isinstance(e.cause, HTTPError) or e.cause.status != 403:
|
||||
raise
|
||||
res = e.cause.response
|
||||
already_impersonating = res.extensions.get('impersonate') is not None
|
||||
if already_impersonating or (
|
||||
res.get_header('cf-mitigated') != 'challenge'
|
||||
and b'<title>Attention Required! | Cloudflare</title>' not in res.read()
|
||||
):
|
||||
raise
|
||||
cf_cookie_domain = traverse_obj(
|
||||
LenientSimpleCookie(e.cause.response.get_header('set-cookie')),
|
||||
('__cf_bm', 'domain'))
|
||||
LenientSimpleCookie(res.get_header('set-cookie')), ('__cf_bm', 'domain'))
|
||||
if cf_cookie_domain:
|
||||
self.write_debug(f'Clearing __cf_bm cookie for {cf_cookie_domain}')
|
||||
self.cookiejar.clear(domain=cf_cookie_domain, path='/', name='__cf_bm')
|
||||
|
||||
@@ -46,6 +46,7 @@ class GofileIE(InfoExtractor):
|
||||
'videopassword': 'password',
|
||||
},
|
||||
}]
|
||||
_STATIC_TOKEN = '4fd6sg89d7s6' # From https://gofile.io/dist/js/config.js
|
||||
_TOKEN = None
|
||||
|
||||
def _real_initialize(self):
|
||||
@@ -60,13 +61,16 @@ class GofileIE(InfoExtractor):
|
||||
self._set_cookie('.gofile.io', 'accountToken', self._TOKEN)
|
||||
|
||||
def _entries(self, file_id):
|
||||
query_params = {'wt': '4fd6sg89d7s6'} # From https://gofile.io/dist/js/alljs.js
|
||||
password = self.get_param('videopassword')
|
||||
if password:
|
||||
query_params = {}
|
||||
if password := self.get_param('videopassword'):
|
||||
query_params['password'] = hashlib.sha256(password.encode()).hexdigest()
|
||||
|
||||
files = self._download_json(
|
||||
f'https://api.gofile.io/contents/{file_id}', file_id, 'Getting filelist',
|
||||
query=query_params, headers={'Authorization': f'Bearer {self._TOKEN}'})
|
||||
query=query_params, headers={
|
||||
'Authorization': f'Bearer {self._TOKEN}',
|
||||
'X-Website-Token': self._STATIC_TOKEN,
|
||||
})
|
||||
|
||||
status = files['status']
|
||||
if status == 'error-passwordRequired':
|
||||
|
||||
@@ -1,21 +1,20 @@
|
||||
import re
|
||||
import urllib.parse
|
||||
|
||||
from .common import InfoExtractor
|
||||
from .youtube import YoutubeIE
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
bug_reports_message,
|
||||
determine_ext,
|
||||
extract_attributes,
|
||||
filter_dict,
|
||||
get_element_by_class,
|
||||
get_element_html_by_id,
|
||||
int_or_none,
|
||||
lowercase_escape,
|
||||
parse_qs,
|
||||
try_get,
|
||||
mimetype2ext,
|
||||
parse_duration,
|
||||
str_or_none,
|
||||
update_url_query,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj, value
|
||||
|
||||
|
||||
class GoogleDriveIE(InfoExtractor):
|
||||
@@ -38,8 +37,8 @@ class GoogleDriveIE(InfoExtractor):
|
||||
'id': '0ByeS4oOUV-49Zzh4R1J6R09zazQ',
|
||||
'ext': 'mp4',
|
||||
'title': 'Big Buck Bunny.mp4',
|
||||
'duration': 45,
|
||||
'thumbnail': 'https://drive.google.com/thumbnail?id=0ByeS4oOUV-49Zzh4R1J6R09zazQ',
|
||||
'duration': 45.069,
|
||||
'thumbnail': r're:https://lh3\.googleusercontent\.com/drive-storage/',
|
||||
},
|
||||
}, {
|
||||
# has itag 50 which is not in YoutubeIE._formats (royalty Free music from 1922)
|
||||
@@ -49,8 +48,29 @@ class GoogleDriveIE(InfoExtractor):
|
||||
'id': '1IP0o8dHcQrIHGgVyp0Ofvx2cGfLzyO1x',
|
||||
'ext': 'mp3',
|
||||
'title': 'My Buddy - Henry Burr - Gus Kahn - Walter Donaldson.mp3',
|
||||
'duration': 184,
|
||||
'thumbnail': 'https://drive.google.com/thumbnail?id=1IP0o8dHcQrIHGgVyp0Ofvx2cGfLzyO1x',
|
||||
'duration': 184.68,
|
||||
},
|
||||
}, {
|
||||
# Has subtitle track
|
||||
'url': 'https://drive.google.com/file/d/1RAGWRgzn85TXCaCk4gxnwF6TGUaZatzE/view',
|
||||
'md5': '05488c528da6ef737ec8c962bfa9724e',
|
||||
'info_dict': {
|
||||
'id': '1RAGWRgzn85TXCaCk4gxnwF6TGUaZatzE',
|
||||
'ext': 'mp4',
|
||||
'title': 'test.mp4',
|
||||
'duration': 9.999,
|
||||
'thumbnail': r're:https://lh3\.googleusercontent\.com/drive-storage/',
|
||||
},
|
||||
}, {
|
||||
# Has subtitle track with kind 'asr'
|
||||
'url': 'https://drive.google.com/file/d/1Prvv9-mtDDfN_gkJgtt1OFvIULK8c3Ev/view',
|
||||
'md5': 'ccae12d07f18b5988900b2c8b92801fc',
|
||||
'info_dict': {
|
||||
'id': '1Prvv9-mtDDfN_gkJgtt1OFvIULK8c3Ev',
|
||||
'ext': 'mp4',
|
||||
'title': 'LEE NA GYUNG-3410-VOICE_MESSAGE.mp4',
|
||||
'duration': 8.766,
|
||||
'thumbnail': r're:https://lh3\.googleusercontent\.com/drive-storage/',
|
||||
},
|
||||
}, {
|
||||
# video can't be watched anonymously due to view count limit reached,
|
||||
@@ -71,17 +91,6 @@ class GoogleDriveIE(InfoExtractor):
|
||||
'url': 'https://drive.usercontent.google.com/download?id=0ByeS4oOUV-49Zzh4R1J6R09zazQ',
|
||||
'only_matching': True,
|
||||
}]
|
||||
_FORMATS_EXT = {
|
||||
**{k: v['ext'] for k, v in YoutubeIE._formats.items() if v.get('ext')},
|
||||
'50': 'm4a',
|
||||
}
|
||||
_BASE_URL_CAPTIONS = 'https://drive.google.com/timedtext'
|
||||
_CAPTIONS_ENTRY_TAG = {
|
||||
'subtitles': 'track',
|
||||
'automatic_captions': 'target',
|
||||
}
|
||||
_caption_formats_ext = []
|
||||
_captions_xml = None
|
||||
|
||||
@classmethod
|
||||
def _extract_embed_urls(cls, url, webpage):
|
||||
@@ -91,129 +100,73 @@ class GoogleDriveIE(InfoExtractor):
|
||||
if mobj:
|
||||
yield 'https://drive.google.com/file/d/{}'.format(mobj.group('id'))
|
||||
|
||||
def _download_subtitles_xml(self, video_id, subtitles_id, hl):
|
||||
if self._captions_xml:
|
||||
return
|
||||
self._captions_xml = self._download_xml(
|
||||
self._BASE_URL_CAPTIONS, video_id, query={
|
||||
'id': video_id,
|
||||
'vid': subtitles_id,
|
||||
'hl': hl,
|
||||
@staticmethod
|
||||
def _construct_subtitle_url(base_url, video_id, language, fmt, kind):
|
||||
return update_url_query(
|
||||
base_url, filter_dict({
|
||||
'hl': 'en-US',
|
||||
'v': video_id,
|
||||
'type': 'track',
|
||||
'lang': language,
|
||||
'fmt': fmt,
|
||||
'kind': kind,
|
||||
}))
|
||||
|
||||
def _get_subtitles(self, video_id, video_info):
|
||||
subtitles = {}
|
||||
timed_text_base_url = traverse_obj(video_info, ('timedTextDetails', 'timedTextBaseUrl', {url_or_none}))
|
||||
if not timed_text_base_url:
|
||||
return subtitles
|
||||
subtitle_data = self._download_xml(
|
||||
timed_text_base_url, video_id, 'Downloading subtitles XML', fatal=False, query={
|
||||
'hl': 'en-US',
|
||||
'type': 'list',
|
||||
'tlangs': '1',
|
||||
'fmts': '1',
|
||||
'vssids': '1',
|
||||
}, note='Downloading subtitles XML',
|
||||
errnote='Unable to download subtitles XML', fatal=False)
|
||||
if self._captions_xml:
|
||||
for f in self._captions_xml.findall('format'):
|
||||
if f.attrib.get('fmt_code') and not f.attrib.get('default'):
|
||||
self._caption_formats_ext.append(f.attrib['fmt_code'])
|
||||
|
||||
def _get_captions_by_type(self, video_id, subtitles_id, caption_type,
|
||||
origin_lang_code=None, origin_lang_name=None):
|
||||
if not subtitles_id or not caption_type:
|
||||
return
|
||||
captions = {}
|
||||
for caption_entry in self._captions_xml.findall(
|
||||
self._CAPTIONS_ENTRY_TAG[caption_type]):
|
||||
caption_lang_code = caption_entry.attrib.get('lang_code')
|
||||
caption_name = caption_entry.attrib.get('name') or origin_lang_name
|
||||
if not caption_lang_code or not caption_name:
|
||||
self.report_warning(f'Missing necessary caption metadata. '
|
||||
f'Need lang_code and name attributes. '
|
||||
f'Found: {caption_entry.attrib}')
|
||||
continue
|
||||
caption_format_data = []
|
||||
for caption_format in self._caption_formats_ext:
|
||||
query = {
|
||||
'vid': subtitles_id,
|
||||
'v': video_id,
|
||||
'fmt': caption_format,
|
||||
'lang': (caption_lang_code if origin_lang_code is None
|
||||
else origin_lang_code),
|
||||
'type': 'track',
|
||||
'name': caption_name,
|
||||
'kind': '',
|
||||
}
|
||||
if origin_lang_code is not None:
|
||||
query.update({'tlang': caption_lang_code})
|
||||
caption_format_data.append({
|
||||
'url': update_url_query(self._BASE_URL_CAPTIONS, query),
|
||||
'ext': caption_format,
|
||||
})
|
||||
captions[caption_lang_code] = caption_format_data
|
||||
return captions
|
||||
|
||||
def _get_subtitles(self, video_id, subtitles_id, hl):
|
||||
if not subtitles_id or not hl:
|
||||
return
|
||||
self._download_subtitles_xml(video_id, subtitles_id, hl)
|
||||
if not self._captions_xml:
|
||||
return
|
||||
return self._get_captions_by_type(video_id, subtitles_id, 'subtitles')
|
||||
|
||||
def _get_automatic_captions(self, video_id, subtitles_id, hl):
|
||||
if not subtitles_id or not hl:
|
||||
return
|
||||
self._download_subtitles_xml(video_id, subtitles_id, hl)
|
||||
if not self._captions_xml:
|
||||
return
|
||||
track = next((t for t in self._captions_xml.findall('track') if t.attrib.get('cantran') == 'true'), None)
|
||||
if track is None:
|
||||
return
|
||||
origin_lang_code = track.attrib.get('lang_code')
|
||||
origin_lang_name = track.attrib.get('name')
|
||||
if not origin_lang_code or not origin_lang_name:
|
||||
return
|
||||
return self._get_captions_by_type(
|
||||
video_id, subtitles_id, 'automatic_captions', origin_lang_code, origin_lang_name)
|
||||
'tlangs': 1,
|
||||
'v': video_id,
|
||||
'vssids': 1,
|
||||
})
|
||||
subtitle_formats = traverse_obj(subtitle_data, (lambda _, v: v.tag == 'format', {lambda x: x.get('fmt_code')}, {str}))
|
||||
for track in traverse_obj(subtitle_data, (lambda _, v: v.tag == 'track' and v.get('lang_code'))):
|
||||
language = track.get('lang_code')
|
||||
subtitles.setdefault(language, []).extend([{
|
||||
'url': self._construct_subtitle_url(
|
||||
timed_text_base_url, video_id, language, sub_fmt, track.get('kind')),
|
||||
'name': track.get('lang_original'),
|
||||
'ext': sub_fmt,
|
||||
} for sub_fmt in subtitle_formats])
|
||||
return subtitles
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
video_info = urllib.parse.parse_qs(self._download_webpage(
|
||||
'https://drive.google.com/get_video_info',
|
||||
video_id, 'Downloading video webpage', query={'docid': video_id}))
|
||||
|
||||
def get_value(key):
|
||||
return try_get(video_info, lambda x: x[key][0])
|
||||
|
||||
reason = get_value('reason')
|
||||
title = get_value('title')
|
||||
video_info = self._download_json(
|
||||
f'https://content-workspacevideo-pa.googleapis.com/v1/drive/media/{video_id}/playback',
|
||||
video_id, 'Downloading video webpage', query={'key': 'AIzaSyDVQw45DwoYh632gvsP5vPDqEKvb-Ywnb8'},
|
||||
headers={'Referer': 'https://drive.google.com/'})
|
||||
|
||||
formats = []
|
||||
fmt_stream_map = (get_value('fmt_stream_map') or '').split(',')
|
||||
fmt_list = (get_value('fmt_list') or '').split(',')
|
||||
if fmt_stream_map and fmt_list:
|
||||
resolutions = {}
|
||||
for fmt in fmt_list:
|
||||
mobj = re.search(
|
||||
r'^(?P<format_id>\d+)/(?P<width>\d+)[xX](?P<height>\d+)', fmt)
|
||||
if mobj:
|
||||
resolutions[mobj.group('format_id')] = (
|
||||
int(mobj.group('width')), int(mobj.group('height')))
|
||||
for fmt in traverse_obj(video_info, (
|
||||
'mediaStreamingData', 'formatStreamingData', ('adaptiveTranscodes', 'progressiveTranscodes'),
|
||||
lambda _, v: url_or_none(v['url']))):
|
||||
formats.append({
|
||||
**traverse_obj(fmt, {
|
||||
'url': 'url',
|
||||
'format_id': ('itag', {int}, {str_or_none}),
|
||||
}),
|
||||
**traverse_obj(fmt, ('transcodeMetadata', {
|
||||
'ext': ('mimeType', {mimetype2ext}),
|
||||
'width': ('width', {int_or_none}),
|
||||
'height': ('height', {int_or_none}),
|
||||
'fps': ('videoFps', {int_or_none}),
|
||||
'filesize': ('contentLength', {int_or_none}),
|
||||
'vcodec': ((('videoCodecString', {str}), {value('none')}), any),
|
||||
'acodec': ((('audioCodecString', {str}), {value('none')}), any),
|
||||
})),
|
||||
'downloader_options': {
|
||||
'http_chunk_size': 10 << 20,
|
||||
},
|
||||
})
|
||||
|
||||
for fmt_stream in fmt_stream_map:
|
||||
fmt_stream_split = fmt_stream.split('|')
|
||||
if len(fmt_stream_split) < 2:
|
||||
continue
|
||||
format_id, format_url = fmt_stream_split[:2]
|
||||
ext = self._FORMATS_EXT.get(format_id)
|
||||
if not ext:
|
||||
self.report_warning(f'Unknown format {format_id}{bug_reports_message()}')
|
||||
f = {
|
||||
'url': lowercase_escape(format_url),
|
||||
'format_id': format_id,
|
||||
'ext': ext,
|
||||
}
|
||||
resolution = resolutions.get(format_id)
|
||||
if resolution:
|
||||
f.update({
|
||||
'width': resolution[0],
|
||||
'height': resolution[1],
|
||||
})
|
||||
formats.append(f)
|
||||
title = traverse_obj(video_info, ('mediaMetadata', 'title', {str}))
|
||||
|
||||
source_url = update_url_query(
|
||||
'https://drive.usercontent.google.com/download', {
|
||||
@@ -264,30 +217,20 @@ class GoogleDriveIE(InfoExtractor):
|
||||
or get_element_by_class('uc-error-caption', confirmation_webpage)
|
||||
or 'unable to extract confirmation code')
|
||||
|
||||
if not formats and reason:
|
||||
if title:
|
||||
self.raise_no_formats(reason, expected=True)
|
||||
else:
|
||||
raise ExtractorError(reason, expected=True)
|
||||
|
||||
hl = get_value('hl')
|
||||
subtitles_id = None
|
||||
ttsurl = get_value('ttsurl')
|
||||
if ttsurl:
|
||||
# the subtitles ID is the vid param of the ttsurl query
|
||||
subtitles_id = parse_qs(ttsurl).get('vid', [None])[-1]
|
||||
|
||||
self.cookiejar.clear(domain='.google.com', path='/', name='NID')
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'thumbnail': 'https://drive.google.com/thumbnail?id=' + video_id,
|
||||
'duration': int_or_none(get_value('length_seconds')),
|
||||
**traverse_obj(video_info, {
|
||||
'duration': ('mediaMetadata', 'duration', {parse_duration}),
|
||||
'thumbnails': ('thumbnails', lambda _, v: url_or_none(v['url']), {
|
||||
'url': 'url',
|
||||
'ext': ('mimeType', {mimetype2ext}),
|
||||
'width': ('width', {int}),
|
||||
'height': ('height', {int}),
|
||||
}),
|
||||
}),
|
||||
'formats': formats,
|
||||
'subtitles': self.extract_subtitles(video_id, subtitles_id, hl),
|
||||
'automatic_captions': self.extract_automatic_captions(
|
||||
video_id, subtitles_id, hl),
|
||||
'subtitles': self.extract_subtitles(video_id, video_info),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -13,12 +13,14 @@ from ..utils.traversal import get_first, traverse_obj
|
||||
|
||||
|
||||
class GoPlayIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(www\.)?goplay\.be/video/([^/?#]+/[^/?#]+/|)(?P<id>[^/#]+)'
|
||||
IE_NAME = 'play.tv'
|
||||
IE_DESC = 'PLAY (formerly goplay.be)'
|
||||
_VALID_URL = r'https?://(www\.)?play\.tv/video/([^/?#]+/[^/?#]+/|)(?P<id>[^/#]+)'
|
||||
|
||||
_NETRC_MACHINE = 'goplay'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://www.goplay.be/video/de-slimste-mens-ter-wereld/de-slimste-mens-ter-wereld-s22/de-slimste-mens-ter-wereld-s22-aflevering-1',
|
||||
'url': 'https://www.play.tv/video/de-slimste-mens-ter-wereld/de-slimste-mens-ter-wereld-s22/de-slimste-mens-ter-wereld-s22-aflevering-1',
|
||||
'info_dict': {
|
||||
'id': '2baa4560-87a0-421b-bffc-359914e3c387',
|
||||
'ext': 'mp4',
|
||||
@@ -33,7 +35,7 @@ class GoPlayIE(InfoExtractor):
|
||||
'params': {'skip_download': True},
|
||||
'skip': 'This video is only available for registered users',
|
||||
}, {
|
||||
'url': 'https://www.goplay.be/video/1917',
|
||||
'url': 'https://www.play.tv/video/1917',
|
||||
'info_dict': {
|
||||
'id': '40cac41d-8d29-4ef5-aa11-75047b9f0907',
|
||||
'ext': 'mp4',
|
||||
@@ -43,7 +45,7 @@ class GoPlayIE(InfoExtractor):
|
||||
'params': {'skip_download': True},
|
||||
'skip': 'This video is only available for registered users',
|
||||
}, {
|
||||
'url': 'https://www.goplay.be/video/de-mol/de-mol-s11/de-mol-s11-aflevering-1#autoplay',
|
||||
'url': 'https://www.play.tv/video/de-mol/de-mol-s11/de-mol-s11-aflevering-1#autoplay',
|
||||
'info_dict': {
|
||||
'id': 'ecb79672-92b9-4cd9-a0d7-e2f0250681ee',
|
||||
'ext': 'mp4',
|
||||
@@ -101,7 +103,7 @@ class GoPlayIE(InfoExtractor):
|
||||
break
|
||||
|
||||
api = self._download_json(
|
||||
f'https://api.goplay.be/web/v1/videos/long-form/{video_id}',
|
||||
f'https://api.play.tv/web/v1/videos/long-form/{video_id}',
|
||||
video_id, headers={
|
||||
'Authorization': f'Bearer {self._id_token}',
|
||||
**self.geo_verification_headers(),
|
||||
|
||||
@@ -27,7 +27,7 @@ class HotStarBaseIE(InfoExtractor):
|
||||
_TOKEN_NAME = 'userUP'
|
||||
_BASE_URL = 'https://www.hotstar.com'
|
||||
_API_URL = 'https://api.hotstar.com'
|
||||
_API_URL_V2 = 'https://apix.hotstar.com/v2'
|
||||
_API_URL_V2 = 'https://www.hotstar.com/api/internal/bff/v2'
|
||||
_AKAMAI_ENCRYPTION_KEY = b'\x05\xfc\x1a\x01\xca\xc9\x4b\xc4\x12\xfc\x53\x12\x07\x75\xf9\xee'
|
||||
|
||||
_FREE_HEADERS = {
|
||||
|
||||
@@ -9,14 +9,12 @@ from .openload import PhantomJSwrapper
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
clean_html,
|
||||
decode_packed_codes,
|
||||
float_or_none,
|
||||
format_field,
|
||||
get_element_by_attribute,
|
||||
get_element_by_id,
|
||||
int_or_none,
|
||||
js_to_json,
|
||||
ohdave_rsa_encrypt,
|
||||
parse_age_limit,
|
||||
parse_duration,
|
||||
parse_iso8601,
|
||||
@@ -33,143 +31,12 @@ def md5_text(text):
|
||||
return hashlib.md5(text.encode()).hexdigest()
|
||||
|
||||
|
||||
class IqiyiSDK:
|
||||
def __init__(self, target, ip, timestamp):
|
||||
self.target = target
|
||||
self.ip = ip
|
||||
self.timestamp = timestamp
|
||||
|
||||
@staticmethod
|
||||
def split_sum(data):
|
||||
return str(sum(int(p, 16) for p in data))
|
||||
|
||||
@staticmethod
|
||||
def digit_sum(num):
|
||||
if isinstance(num, int):
|
||||
num = str(num)
|
||||
return str(sum(map(int, num)))
|
||||
|
||||
def even_odd(self):
|
||||
even = self.digit_sum(str(self.timestamp)[::2])
|
||||
odd = self.digit_sum(str(self.timestamp)[1::2])
|
||||
return even, odd
|
||||
|
||||
def preprocess(self, chunksize):
|
||||
self.target = md5_text(self.target)
|
||||
chunks = []
|
||||
for i in range(32 // chunksize):
|
||||
chunks.append(self.target[chunksize * i:chunksize * (i + 1)])
|
||||
if 32 % chunksize:
|
||||
chunks.append(self.target[32 - 32 % chunksize:])
|
||||
return chunks, list(map(int, self.ip.split('.')))
|
||||
|
||||
def mod(self, modulus):
|
||||
chunks, ip = self.preprocess(32)
|
||||
self.target = chunks[0] + ''.join(str(p % modulus) for p in ip)
|
||||
|
||||
def split(self, chunksize):
|
||||
modulus_map = {
|
||||
4: 256,
|
||||
5: 10,
|
||||
8: 100,
|
||||
}
|
||||
|
||||
chunks, ip = self.preprocess(chunksize)
|
||||
ret = ''
|
||||
for i in range(len(chunks)):
|
||||
ip_part = str(ip[i] % modulus_map[chunksize]) if i < 4 else ''
|
||||
if chunksize == 8:
|
||||
ret += ip_part + chunks[i]
|
||||
else:
|
||||
ret += chunks[i] + ip_part
|
||||
self.target = ret
|
||||
|
||||
def handle_input16(self):
|
||||
self.target = md5_text(self.target)
|
||||
self.target = self.split_sum(self.target[:16]) + self.target + self.split_sum(self.target[16:])
|
||||
|
||||
def handle_input8(self):
|
||||
self.target = md5_text(self.target)
|
||||
ret = ''
|
||||
for i in range(4):
|
||||
part = self.target[8 * i:8 * (i + 1)]
|
||||
ret += self.split_sum(part) + part
|
||||
self.target = ret
|
||||
|
||||
def handleSum(self):
|
||||
self.target = md5_text(self.target)
|
||||
self.target = self.split_sum(self.target) + self.target
|
||||
|
||||
def date(self, scheme):
|
||||
self.target = md5_text(self.target)
|
||||
d = time.localtime(self.timestamp)
|
||||
strings = {
|
||||
'y': str(d.tm_year),
|
||||
'm': '%02d' % d.tm_mon,
|
||||
'd': '%02d' % d.tm_mday,
|
||||
}
|
||||
self.target += ''.join(strings[c] for c in scheme)
|
||||
|
||||
def split_time_even_odd(self):
|
||||
even, odd = self.even_odd()
|
||||
self.target = odd + md5_text(self.target) + even
|
||||
|
||||
def split_time_odd_even(self):
|
||||
even, odd = self.even_odd()
|
||||
self.target = even + md5_text(self.target) + odd
|
||||
|
||||
def split_ip_time_sum(self):
|
||||
chunks, ip = self.preprocess(32)
|
||||
self.target = str(sum(ip)) + chunks[0] + self.digit_sum(self.timestamp)
|
||||
|
||||
def split_time_ip_sum(self):
|
||||
chunks, ip = self.preprocess(32)
|
||||
self.target = self.digit_sum(self.timestamp) + chunks[0] + str(sum(ip))
|
||||
|
||||
|
||||
class IqiyiSDKInterpreter:
|
||||
def __init__(self, sdk_code):
|
||||
self.sdk_code = sdk_code
|
||||
|
||||
def run(self, target, ip, timestamp):
|
||||
self.sdk_code = decode_packed_codes(self.sdk_code)
|
||||
|
||||
functions = re.findall(r'input=([a-zA-Z0-9]+)\(input', self.sdk_code)
|
||||
|
||||
sdk = IqiyiSDK(target, ip, timestamp)
|
||||
|
||||
other_functions = {
|
||||
'handleSum': sdk.handleSum,
|
||||
'handleInput8': sdk.handle_input8,
|
||||
'handleInput16': sdk.handle_input16,
|
||||
'splitTimeEvenOdd': sdk.split_time_even_odd,
|
||||
'splitTimeOddEven': sdk.split_time_odd_even,
|
||||
'splitIpTimeSum': sdk.split_ip_time_sum,
|
||||
'splitTimeIpSum': sdk.split_time_ip_sum,
|
||||
}
|
||||
for function in functions:
|
||||
if re.match(r'mod\d+', function):
|
||||
sdk.mod(int(function[3:]))
|
||||
elif re.match(r'date[ymd]{3}', function):
|
||||
sdk.date(function[4:])
|
||||
elif re.match(r'split\d+', function):
|
||||
sdk.split(int(function[5:]))
|
||||
elif function in other_functions:
|
||||
other_functions[function]()
|
||||
else:
|
||||
raise ExtractorError(f'Unknown function {function}')
|
||||
|
||||
return sdk.target
|
||||
|
||||
|
||||
class IqiyiIE(InfoExtractor):
|
||||
IE_NAME = 'iqiyi'
|
||||
IE_DESC = '爱奇艺'
|
||||
|
||||
_VALID_URL = r'https?://(?:(?:[^.]+\.)?iqiyi\.com|www\.pps\.tv)/.+\.html'
|
||||
|
||||
_NETRC_MACHINE = 'iqiyi'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
|
||||
# MD5 checksum differs on my machine and Travis CI
|
||||
@@ -234,57 +101,6 @@ class IqiyiIE(InfoExtractor):
|
||||
'18': 7, # 1080p
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _rsa_fun(data):
|
||||
# public key extracted from http://static.iqiyi.com/js/qiyiV2/20160129180840/jobs/i18n/i18nIndex.js
|
||||
N = 0xab86b6371b5318aaa1d3c9e612a9f1264f372323c8c0f19875b5fc3b3fd3afcc1e5bec527aa94bfa85bffc157e4245aebda05389a5357b75115ac94f074aefcd
|
||||
e = 65537
|
||||
|
||||
return ohdave_rsa_encrypt(data, e, N)
|
||||
|
||||
def _perform_login(self, username, password):
|
||||
|
||||
data = self._download_json(
|
||||
'http://kylin.iqiyi.com/get_token', None,
|
||||
note='Get token for logging', errnote='Unable to get token for logging')
|
||||
sdk = data['sdk']
|
||||
timestamp = int(time.time())
|
||||
target = (
|
||||
f'/apis/reglogin/login.action?lang=zh_TW&area_code=null&email={username}'
|
||||
f'&passwd={self._rsa_fun(password.encode())}&agenttype=1&from=undefined&keeplogin=0&piccode=&fromurl=&_pos=1')
|
||||
|
||||
interp = IqiyiSDKInterpreter(sdk)
|
||||
sign = interp.run(target, data['ip'], timestamp)
|
||||
|
||||
validation_params = {
|
||||
'target': target,
|
||||
'server': 'BEA3AA1908656AABCCFF76582C4C6660',
|
||||
'token': data['token'],
|
||||
'bird_src': 'f8d91d57af224da7893dd397d52d811a',
|
||||
'sign': sign,
|
||||
'bird_t': timestamp,
|
||||
}
|
||||
validation_result = self._download_json(
|
||||
'http://kylin.iqiyi.com/validate?' + urllib.parse.urlencode(validation_params), None,
|
||||
note='Validate credentials', errnote='Unable to validate credentials')
|
||||
|
||||
MSG_MAP = {
|
||||
'P00107': 'please login via the web interface and enter the CAPTCHA code',
|
||||
'P00117': 'bad username or password',
|
||||
}
|
||||
|
||||
code = validation_result['code']
|
||||
if code != 'A00000':
|
||||
msg = MSG_MAP.get(code)
|
||||
if not msg:
|
||||
msg = f'error {code}'
|
||||
if validation_result.get('msg'):
|
||||
msg += ': ' + validation_result['msg']
|
||||
self.report_warning('unable to log in: ' + msg)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def get_raw_data(self, tvid, video_id):
|
||||
tm = int(time.time() * 1000)
|
||||
|
||||
|
||||
@@ -98,7 +98,7 @@ class JTBCIE(InfoExtractor):
|
||||
|
||||
formats = []
|
||||
for stream_url in traverse_obj(playback_data, ('sources', 'HLS', ..., 'file', {url_or_none})):
|
||||
stream_url = re.sub(r'/playlist(?:_pd\d+)?\.m3u8', '/index.m3u8', stream_url)
|
||||
stream_url = re.sub(r'/playlist_pd\d+\.m3u8', '/playlist.m3u8', stream_url)
|
||||
formats.extend(self._extract_m3u8_formats(stream_url, video_id, fatal=False))
|
||||
|
||||
metadata = self._download_json(
|
||||
|
||||
@@ -17,57 +17,60 @@ class KikaIE(InfoExtractor):
|
||||
_GEO_COUNTRIES = ['DE']
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://www.kika.de/logo/videos/logo-vom-samstag-einunddreissig-august-zweitausendvierundzwanzig-100',
|
||||
'md5': 'fbfc8da483719ef06f396e5e5b938c69',
|
||||
# Video without season/episode info
|
||||
'url': 'https://www.kika.de/logo/videos/logo-vom-dienstag-achtundzwanzig-oktober-zweitausendfuenfundzwanzig-100',
|
||||
'md5': '4a9f6e0f9c6bfcc82394c294f186d6db',
|
||||
'info_dict': {
|
||||
'id': 'logo-vom-samstag-einunddreissig-august-zweitausendvierundzwanzig-100',
|
||||
'id': 'logo-vom-dienstag-achtundzwanzig-oktober-zweitausendfuenfundzwanzig-100',
|
||||
'ext': 'mp4',
|
||||
'upload_date': '20240831',
|
||||
'timestamp': 1725126600,
|
||||
'season_number': 2024,
|
||||
'modified_date': '20240831',
|
||||
'episode': 'Episode 476',
|
||||
'episode_number': 476,
|
||||
'season': 'Season 2024',
|
||||
'duration': 634,
|
||||
'title': 'logo! vom Samstag, 31. August 2024',
|
||||
'modified_timestamp': 1725129983,
|
||||
'title': 'logo! vom Dienstag, 28. Oktober 2025',
|
||||
'description': 'md5:4d28b92cef423bec99740ffaa3e7ec04',
|
||||
'duration': 651,
|
||||
'timestamp': 1761678000,
|
||||
'upload_date': '20251028',
|
||||
'modified_timestamp': 1761682624,
|
||||
'modified_date': '20251028',
|
||||
},
|
||||
}, {
|
||||
# Video with season/episode info
|
||||
# Also: Video with subtitles
|
||||
'url': 'https://www.kika.de/kaltstart/videos/video92498',
|
||||
'md5': '710ece827e5055094afeb474beacb7aa',
|
||||
'md5': 'e58073070acb195906c55c4ad31dceb3',
|
||||
'info_dict': {
|
||||
'id': 'video92498',
|
||||
'ext': 'mp4',
|
||||
'title': '7. Wo ist Leo?',
|
||||
'description': 'md5:fb48396a5b75068bcac1df74f1524920',
|
||||
'duration': 436,
|
||||
'season': 'Season 1',
|
||||
'season_number': 1,
|
||||
'episode': 'Episode 7',
|
||||
'episode_number': 7,
|
||||
'timestamp': 1702926876,
|
||||
'upload_date': '20231218',
|
||||
'episode_number': 7,
|
||||
'modified_date': '20240319',
|
||||
'modified_timestamp': 1710880610,
|
||||
'episode': 'Episode 7',
|
||||
'season_number': 1,
|
||||
'season': 'Season 1',
|
||||
'modified_date': '20240319',
|
||||
'subtitles': 'count:1',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.kika.de/bernd-das-brot/astrobrot/videos/video90088',
|
||||
'md5': 'ffd1b700d7de0a6616a1d08544c77294',
|
||||
# Video without subtitles
|
||||
'url': 'https://www.kika.de/die-pfefferkoerner/videos/abgezogen-102',
|
||||
'md5': '62e97961ce5343c19f0f330a1b6dd736',
|
||||
'info_dict': {
|
||||
'id': 'video90088',
|
||||
'id': 'abgezogen-102',
|
||||
'ext': 'mp4',
|
||||
'upload_date': '20221102',
|
||||
'timestamp': 1667390580,
|
||||
'duration': 197,
|
||||
'modified_timestamp': 1711093771,
|
||||
'episode_number': 8,
|
||||
'title': 'Es ist nicht leicht, ein Astrobrot zu sein',
|
||||
'modified_date': '20240322',
|
||||
'description': 'md5:d3641deaf1b5515a160788b2be4159a9',
|
||||
'season_number': 1,
|
||||
'episode': 'Episode 8',
|
||||
'title': '1. Abgezogen',
|
||||
'description': 'md5:42d87963364391f9f8eba8affcb30bd2',
|
||||
'duration': 1574,
|
||||
'season': 'Season 1',
|
||||
'season_number': 1,
|
||||
'episode': 'Episode 1',
|
||||
'episode_number': 1,
|
||||
'timestamp': 1735382700,
|
||||
'upload_date': '20241228',
|
||||
'modified_timestamp': 1757344051,
|
||||
'modified_date': '20250908',
|
||||
'subtitles': 'count:0',
|
||||
},
|
||||
}]
|
||||
|
||||
@@ -78,16 +81,19 @@ class KikaIE(InfoExtractor):
|
||||
video_assets = self._download_json(doc['assets']['url'], video_id)
|
||||
|
||||
subtitles = {}
|
||||
if ttml_resource := url_or_none(video_assets.get('videoSubtitle')):
|
||||
subtitles['de'] = [{
|
||||
'url': ttml_resource,
|
||||
'ext': 'ttml',
|
||||
}]
|
||||
if webvtt_resource := url_or_none(video_assets.get('webvttUrl')):
|
||||
subtitles.setdefault('de', []).append({
|
||||
'url': webvtt_resource,
|
||||
'ext': 'vtt',
|
||||
})
|
||||
# Subtitle API endpoints may be present in the JSON even if there are no subtitles.
|
||||
# They then return HTTP 200 with invalid data. So we must check explicitly.
|
||||
if doc.get('hasSubtitle'):
|
||||
if ttml_resource := url_or_none(video_assets.get('videoSubtitle')):
|
||||
subtitles['de'] = [{
|
||||
'url': ttml_resource,
|
||||
'ext': 'ttml',
|
||||
}]
|
||||
if webvtt_resource := url_or_none(video_assets.get('webvttUrl')):
|
||||
subtitles.setdefault('de', []).append({
|
||||
'url': webvtt_resource,
|
||||
'ext': 'vtt',
|
||||
})
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
|
||||
@@ -8,12 +8,10 @@ from ..utils import (
|
||||
ExtractorError,
|
||||
determine_ext,
|
||||
filter_dict,
|
||||
get_first,
|
||||
int_or_none,
|
||||
parse_iso8601,
|
||||
update_url,
|
||||
url_or_none,
|
||||
variadic,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
@@ -51,7 +49,7 @@ class LoomIE(InfoExtractor):
|
||||
}, {
|
||||
# m3u8 raw-url, mp4 transcoded-url, cdn url == raw-url, vtt sub and json subs
|
||||
'url': 'https://www.loom.com/share/9458bcbf79784162aa62ffb8dd66201b',
|
||||
'md5': '51737ec002969dd28344db4d60b9cbbb',
|
||||
'md5': '7b6bfdef8181c4ffc376e18919a4dcc2',
|
||||
'info_dict': {
|
||||
'id': '9458bcbf79784162aa62ffb8dd66201b',
|
||||
'ext': 'mp4',
|
||||
@@ -71,12 +69,13 @@ class LoomIE(InfoExtractor):
|
||||
'ext': 'webm',
|
||||
'title': 'OMFG clown',
|
||||
'description': 'md5:285c5ee9d62aa087b7e3271b08796815',
|
||||
'uploader': 'MrPumkin B',
|
||||
'uploader': 'Brailey Bragg',
|
||||
'upload_date': '20210924',
|
||||
'timestamp': 1632519618,
|
||||
'duration': 210,
|
||||
},
|
||||
'params': {'skip_download': 'dash'},
|
||||
'expected_warnings': ['Failed to parse JSON'], # transcoded-url no longer available
|
||||
}, {
|
||||
# password-protected
|
||||
'url': 'https://www.loom.com/share/50e26e8aeb7940189dff5630f95ce1f4',
|
||||
@@ -91,10 +90,11 @@ class LoomIE(InfoExtractor):
|
||||
'duration': 35,
|
||||
},
|
||||
'params': {'videopassword': 'seniorinfants2'},
|
||||
'expected_warnings': ['Failed to parse JSON'], # transcoded-url no longer available
|
||||
}, {
|
||||
# embed, transcoded-url endpoint sends empty JSON response, split video and audio HLS formats
|
||||
'url': 'https://www.loom.com/embed/ddcf1c1ad21f451ea7468b1e33917e4e',
|
||||
'md5': 'b321d261656848c184a94e3b93eae28d',
|
||||
'md5': 'f983a0f02f24331738b2f43aecb05256',
|
||||
'info_dict': {
|
||||
'id': 'ddcf1c1ad21f451ea7468b1e33917e4e',
|
||||
'ext': 'mp4',
|
||||
@@ -119,11 +119,12 @@ class LoomIE(InfoExtractor):
|
||||
'duration': 247,
|
||||
'timestamp': 1676274030,
|
||||
},
|
||||
'skip': '404 Not Found',
|
||||
}]
|
||||
|
||||
_GRAPHQL_VARIABLES = {
|
||||
'GetVideoSource': {
|
||||
'acceptableMimes': ['DASH', 'M3U8', 'MP4'],
|
||||
'acceptableMimes': ['DASH', 'M3U8', 'MP4', 'WEBM'],
|
||||
},
|
||||
}
|
||||
_GRAPHQL_QUERIES = {
|
||||
@@ -192,6 +193,12 @@ class LoomIE(InfoExtractor):
|
||||
id
|
||||
nullableRawCdnUrl(acceptableMimes: $acceptableMimes, password: $password) {
|
||||
url
|
||||
credentials {
|
||||
Policy
|
||||
Signature
|
||||
KeyPairId
|
||||
__typename
|
||||
}
|
||||
__typename
|
||||
}
|
||||
__typename
|
||||
@@ -240,9 +247,9 @@ class LoomIE(InfoExtractor):
|
||||
}
|
||||
}\n'''),
|
||||
}
|
||||
_APOLLO_GRAPHQL_VERSION = '0a1856c'
|
||||
_APOLLO_GRAPHQL_VERSION = '45a5bd4'
|
||||
|
||||
def _call_graphql_api(self, operations, video_id, note=None, errnote=None):
|
||||
def _call_graphql_api(self, operation_name, video_id, note=None, errnote=None, fatal=True):
|
||||
password = self.get_param('videopassword')
|
||||
return self._download_json(
|
||||
'https://www.loom.com/graphql', video_id, note or 'Downloading GraphQL JSON',
|
||||
@@ -252,7 +259,9 @@ class LoomIE(InfoExtractor):
|
||||
'x-loom-request-source': f'loom_web_{self._APOLLO_GRAPHQL_VERSION}',
|
||||
'apollographql-client-name': 'web',
|
||||
'apollographql-client-version': self._APOLLO_GRAPHQL_VERSION,
|
||||
}, data=json.dumps([{
|
||||
'graphql-operation-name': operation_name,
|
||||
'Origin': 'https://www.loom.com',
|
||||
}, data=json.dumps({
|
||||
'operationName': operation_name,
|
||||
'variables': {
|
||||
'videoId': video_id,
|
||||
@@ -260,7 +269,7 @@ class LoomIE(InfoExtractor):
|
||||
**self._GRAPHQL_VARIABLES.get(operation_name, {}),
|
||||
},
|
||||
'query': self._GRAPHQL_QUERIES[operation_name],
|
||||
} for operation_name in variadic(operations)], separators=(',', ':')).encode())
|
||||
}, separators=(',', ':')).encode(), fatal=fatal)
|
||||
|
||||
def _call_url_api(self, endpoint, video_id):
|
||||
response = self._download_json(
|
||||
@@ -275,7 +284,7 @@ class LoomIE(InfoExtractor):
|
||||
}, separators=(',', ':')).encode())
|
||||
return traverse_obj(response, ('url', {url_or_none}))
|
||||
|
||||
def _extract_formats(self, video_id, metadata, gql_data):
|
||||
def _extract_formats(self, video_id, metadata, video_data):
|
||||
formats = []
|
||||
video_properties = traverse_obj(metadata, ('video_properties', {
|
||||
'width': ('width', {int_or_none}),
|
||||
@@ -330,7 +339,7 @@ class LoomIE(InfoExtractor):
|
||||
transcoded_url = self._call_url_api('transcoded-url', video_id)
|
||||
formats.extend(get_formats(transcoded_url, 'transcoded', quality=-1)) # transcoded quality
|
||||
|
||||
cdn_url = get_first(gql_data, ('data', 'getVideo', 'nullableRawCdnUrl', 'url', {url_or_none}))
|
||||
cdn_url = traverse_obj(video_data, ('data', 'getVideo', 'nullableRawCdnUrl', 'url', {url_or_none}))
|
||||
# cdn_url is usually a dupe, but the raw-url/transcoded-url endpoints could return errors
|
||||
valid_urls = [update_url(url, query=None) for url in (raw_url, transcoded_url) if url]
|
||||
if cdn_url and update_url(cdn_url, query=None) not in valid_urls:
|
||||
@@ -338,10 +347,21 @@ class LoomIE(InfoExtractor):
|
||||
|
||||
return formats
|
||||
|
||||
def _get_subtitles(self, video_id):
|
||||
subs_data = self._call_graphql_api(
|
||||
'FetchVideoTranscript', video_id, 'Downloading GraphQL subtitles JSON', fatal=False)
|
||||
return filter_dict({
|
||||
'en': traverse_obj(subs_data, (
|
||||
'data', 'fetchVideoTranscript',
|
||||
('source_url', 'captions_source_url'), {
|
||||
'url': {url_or_none},
|
||||
})) or None,
|
||||
})
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
metadata = get_first(
|
||||
self._call_graphql_api('GetVideoSSR', video_id, 'Downloading GraphQL metadata JSON'),
|
||||
metadata = traverse_obj(
|
||||
self._call_graphql_api('GetVideoSSR', video_id, 'Downloading GraphQL metadata JSON', fatal=False),
|
||||
('data', 'getVideo', {dict})) or {}
|
||||
|
||||
if metadata.get('__typename') == 'VideoPasswordMissingOrIncorrect':
|
||||
@@ -350,22 +370,19 @@ class LoomIE(InfoExtractor):
|
||||
'This video is password-protected, use the --video-password option', expected=True)
|
||||
raise ExtractorError('Invalid video password', expected=True)
|
||||
|
||||
gql_data = self._call_graphql_api(['FetchChapters', 'FetchVideoTranscript', 'GetVideoSource'], video_id)
|
||||
video_data = self._call_graphql_api(
|
||||
'GetVideoSource', video_id, 'Downloading GraphQL video JSON')
|
||||
chapter_data = self._call_graphql_api(
|
||||
'FetchChapters', video_id, 'Downloading GraphQL chapters JSON', fatal=False)
|
||||
duration = traverse_obj(metadata, ('video_properties', 'duration', {int_or_none}))
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'duration': duration,
|
||||
'chapters': self._extract_chapters_from_description(
|
||||
get_first(gql_data, ('data', 'fetchVideoChapters', 'content', {str})), duration) or None,
|
||||
'formats': self._extract_formats(video_id, metadata, gql_data),
|
||||
'subtitles': filter_dict({
|
||||
'en': traverse_obj(gql_data, (
|
||||
..., 'data', 'fetchVideoTranscript',
|
||||
('source_url', 'captions_source_url'), {
|
||||
'url': {url_or_none},
|
||||
})) or None,
|
||||
}),
|
||||
traverse_obj(chapter_data, ('data', 'fetchVideoChapters', 'content', {str})), duration) or None,
|
||||
'formats': self._extract_formats(video_id, metadata, video_data),
|
||||
'subtitles': self.extract_subtitles(video_id),
|
||||
**traverse_obj(metadata, {
|
||||
'title': ('name', {str}),
|
||||
'description': ('description', {str}),
|
||||
@@ -376,6 +393,7 @@ class LoomIE(InfoExtractor):
|
||||
|
||||
|
||||
class LoomFolderIE(InfoExtractor):
|
||||
_WORKING = False
|
||||
IE_NAME = 'loom:folder'
|
||||
_VALID_URL = r'https?://(?:www\.)?loom\.com/share/folder/(?P<id>[\da-f]{32})'
|
||||
_TESTS = [{
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import clean_html, int_or_none, traverse_obj
|
||||
|
||||
_API_URL = 'https://dak1vd5vmi7x6.cloudfront.net/api/v1/publicrole/{}/{}?id={}'
|
||||
|
||||
|
||||
class ManotoTVIE(InfoExtractor):
|
||||
IE_DESC = 'Manoto TV (Episode)'
|
||||
_VALID_URL = r'https?://(?:www\.)?manototv\.com/episode/(?P<id>[0-9]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.manototv.com/episode/8475',
|
||||
'info_dict': {
|
||||
'id': '8475',
|
||||
'series': 'خانه های رویایی با برادران اسکات',
|
||||
'season_number': 7,
|
||||
'episode_number': 25,
|
||||
'episode_id': 'My Dream Home S7: Carol & John',
|
||||
'duration': 3600,
|
||||
'categories': ['سرگرمی'],
|
||||
'title': 'کارول و جان',
|
||||
'description': 'md5:d0fff1f8ba5c6775d312a00165d1a97e',
|
||||
'thumbnail': r're:^https?://.*\.(jpeg|png|jpg)$',
|
||||
'ext': 'mp4',
|
||||
},
|
||||
'params': {
|
||||
'skip_download': 'm3u8',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.manototv.com/episode/12576',
|
||||
'info_dict': {
|
||||
'id': '12576',
|
||||
'series': 'فیلم های ایرانی',
|
||||
'episode_id': 'Seh Mah Taatili',
|
||||
'duration': 5400,
|
||||
'view_count': int,
|
||||
'categories': ['سرگرمی'],
|
||||
'title': 'سه ماه تعطیلی',
|
||||
'description': 'سه ماه تعطیلی فیلمی به کارگردانی و نویسندگی شاپور قریب ساختهٔ سال ۱۳۵۶ است.',
|
||||
'thumbnail': r're:^https?://.*\.(jpeg|png|jpg)$',
|
||||
'ext': 'mp4',
|
||||
},
|
||||
'params': {
|
||||
'skip_download': 'm3u8',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
episode_json = self._download_json(_API_URL.format('showmodule', 'episodedetails', video_id), video_id)
|
||||
details = episode_json.get('details', {})
|
||||
formats = self._extract_m3u8_formats(details.get('videoM3u8Url'), video_id, 'mp4')
|
||||
return {
|
||||
'id': video_id,
|
||||
'series': details.get('showTitle'),
|
||||
'season_number': int_or_none(details.get('analyticsSeasonNumber')),
|
||||
'episode_number': int_or_none(details.get('episodeNumber')),
|
||||
'episode_id': details.get('analyticsEpisodeTitle'),
|
||||
'duration': int_or_none(details.get('durationInMinutes'), invscale=60),
|
||||
'view_count': details.get('viewCount'),
|
||||
'categories': [details.get('videoCategory')],
|
||||
'title': details.get('episodeTitle'),
|
||||
'description': clean_html(details.get('episodeDescription')),
|
||||
'thumbnail': details.get('episodelandscapeImgIxUrl'),
|
||||
'formats': formats,
|
||||
}
|
||||
|
||||
|
||||
class ManotoTVShowIE(InfoExtractor):
|
||||
IE_DESC = 'Manoto TV (Show)'
|
||||
_VALID_URL = r'https?://(?:www\.)?manototv\.com/show/(?P<id>[0-9]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.manototv.com/show/2526',
|
||||
'playlist_mincount': 68,
|
||||
'info_dict': {
|
||||
'id': '2526',
|
||||
'title': 'فیلم های ایرانی',
|
||||
'description': 'مجموعه ای از فیلم های سینمای کلاسیک ایران',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
show_id = self._match_id(url)
|
||||
show_json = self._download_json(_API_URL.format('showmodule', 'details', show_id), show_id)
|
||||
show_details = show_json.get('details', {})
|
||||
title = show_details.get('showTitle')
|
||||
description = show_details.get('showSynopsis')
|
||||
|
||||
series_json = self._download_json(_API_URL.format('showmodule', 'serieslist', show_id), show_id)
|
||||
playlist_id = str(traverse_obj(series_json, ('details', 'list', 0, 'id')))
|
||||
|
||||
playlist_json = self._download_json(_API_URL.format('showmodule', 'episodelist', playlist_id), playlist_id)
|
||||
playlist = traverse_obj(playlist_json, ('details', 'list')) or []
|
||||
|
||||
entries = [
|
||||
self.url_result(
|
||||
'https://www.manototv.com/episode/{}'.format(item['slideID']), ie=ManotoTVIE.ie_key(), video_id=item['slideID'])
|
||||
for item in playlist]
|
||||
return self.playlist_result(entries, show_id, title, description)
|
||||
|
||||
|
||||
class ManotoTVLiveIE(InfoExtractor):
|
||||
IE_DESC = 'Manoto TV (Live)'
|
||||
_VALID_URL = r'https?://(?:www\.)?manototv\.com/live/'
|
||||
_TEST = {
|
||||
'url': 'https://www.manototv.com/live/',
|
||||
'info_dict': {
|
||||
'id': 'live',
|
||||
'title': 'Manoto TV Live',
|
||||
'ext': 'mp4',
|
||||
'is_live': True,
|
||||
},
|
||||
'params': {
|
||||
'skip_download': 'm3u8',
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = 'live'
|
||||
json = self._download_json(_API_URL.format('livemodule', 'details', ''), video_id)
|
||||
details = json.get('details', {})
|
||||
video_url = details.get('liveUrl')
|
||||
formats = self._extract_m3u8_formats(video_url, video_id, 'mp4', live=True)
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': 'Manoto TV Live',
|
||||
'is_live': True,
|
||||
'formats': formats,
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
import re
|
||||
import functools
|
||||
import math
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
InAdvancePagedList,
|
||||
clean_html,
|
||||
int_or_none,
|
||||
parse_iso8601,
|
||||
@@ -10,15 +12,64 @@ from ..utils import (
|
||||
from ..utils.traversal import require, traverse_obj
|
||||
|
||||
|
||||
class MaveIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?P<channel>[\w-]+)\.mave\.digital/(?P<id>ep-\d+)'
|
||||
class MaveBaseIE(InfoExtractor):
|
||||
_API_BASE_URL = 'https://api.mave.digital/v1/website'
|
||||
_API_BASE_STORAGE_URL = 'https://store.cloud.mts.ru/mave/'
|
||||
|
||||
def _load_channel_meta(self, channel_id, display_id):
|
||||
return traverse_obj(self._download_json(
|
||||
f'{self._API_BASE_URL}/{channel_id}/', display_id,
|
||||
note='Downloading channel metadata'), 'podcast')
|
||||
|
||||
def _load_episode_meta(self, channel_id, episode_code, display_id):
|
||||
return self._download_json(
|
||||
f'{self._API_BASE_URL}/{channel_id}/episodes/{episode_code}',
|
||||
display_id, note='Downloading episode metadata')
|
||||
|
||||
def _create_entry(self, channel_id, channel_meta, episode_meta):
|
||||
episode_code = traverse_obj(episode_meta, ('code', {int}, {require('episode code')}))
|
||||
return {
|
||||
'display_id': f'{channel_id}-{episode_code}',
|
||||
'extractor_key': MaveIE.ie_key(),
|
||||
'extractor': MaveIE.IE_NAME,
|
||||
'webpage_url': f'https://{channel_id}.mave.digital/ep-{episode_code}',
|
||||
'channel_id': channel_id,
|
||||
'channel_url': f'https://{channel_id}.mave.digital/',
|
||||
'vcodec': 'none',
|
||||
**traverse_obj(episode_meta, {
|
||||
'id': ('id', {str}),
|
||||
'url': ('audio', {urljoin(self._API_BASE_STORAGE_URL)}),
|
||||
'title': ('title', {str}),
|
||||
'description': ('description', {clean_html}),
|
||||
'thumbnail': ('image', {urljoin(self._API_BASE_STORAGE_URL)}),
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'season_number': ('season', {int_or_none}),
|
||||
'episode_number': ('number', {int_or_none}),
|
||||
'view_count': ('listenings', {int_or_none}),
|
||||
'like_count': ('reactions', lambda _, v: v['type'] == 'like', 'count', {int_or_none}, any),
|
||||
'dislike_count': ('reactions', lambda _, v: v['type'] == 'dislike', 'count', {int_or_none}, any),
|
||||
'age_limit': ('is_explicit', {bool}, {lambda x: 18 if x else None}),
|
||||
'timestamp': ('publish_date', {parse_iso8601}),
|
||||
}),
|
||||
**traverse_obj(channel_meta, {
|
||||
'series_id': ('id', {str}),
|
||||
'series': ('title', {str}),
|
||||
'channel': ('title', {str}),
|
||||
'uploader': ('author', {str}),
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
class MaveIE(MaveBaseIE):
|
||||
IE_NAME = 'mave'
|
||||
_VALID_URL = r'https?://(?P<channel_id>[\w-]+)\.mave\.digital/ep-(?P<episode_code>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://ochenlichnoe.mave.digital/ep-25',
|
||||
'md5': 'aa3e513ef588b4366df1520657cbc10c',
|
||||
'info_dict': {
|
||||
'id': '4035f587-914b-44b6-aa5a-d76685ad9bc2',
|
||||
'ext': 'mp3',
|
||||
'display_id': 'ochenlichnoe-ep-25',
|
||||
'display_id': 'ochenlichnoe-25',
|
||||
'title': 'Между мной и миром: психология самооценки',
|
||||
'description': 'md5:4b7463baaccb6982f326bce5c700382a',
|
||||
'uploader': 'Самарский университет',
|
||||
@@ -45,7 +96,7 @@ class MaveIE(InfoExtractor):
|
||||
'info_dict': {
|
||||
'id': '41898bb5-ff57-4797-9236-37a8e537aa21',
|
||||
'ext': 'mp3',
|
||||
'display_id': 'budem-ep-12',
|
||||
'display_id': 'budem-12',
|
||||
'title': 'Екатерина Михайлова: "Горе от ума" не про женщин написана',
|
||||
'description': 'md5:fa3bdd59ee829dfaf16e3efcb13f1d19',
|
||||
'uploader': 'Полина Цветкова+Евгения Акопова',
|
||||
@@ -68,40 +119,72 @@ class MaveIE(InfoExtractor):
|
||||
'upload_date': '20241230',
|
||||
},
|
||||
}]
|
||||
_API_BASE_URL = 'https://api.mave.digital/'
|
||||
|
||||
def _real_extract(self, url):
|
||||
channel_id, slug = self._match_valid_url(url).group('channel', 'id')
|
||||
display_id = f'{channel_id}-{slug}'
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
data = traverse_obj(
|
||||
self._search_nuxt_json(webpage, display_id),
|
||||
('data', lambda _, v: v['activeEpisodeData'], any, {require('podcast data')}))
|
||||
channel_id, episode_code = self._match_valid_url(url).group(
|
||||
'channel_id', 'episode_code')
|
||||
display_id = f'{channel_id}-{episode_code}'
|
||||
|
||||
channel_meta = self._load_channel_meta(channel_id, display_id)
|
||||
episode_meta = self._load_episode_meta(channel_id, episode_code, display_id)
|
||||
|
||||
return self._create_entry(channel_id, channel_meta, episode_meta)
|
||||
|
||||
|
||||
class MaveChannelIE(MaveBaseIE):
|
||||
IE_NAME = 'mave:channel'
|
||||
_VALID_URL = r'https?://(?P<id>[\w-]+)\.mave\.digital/?(?:$|[?#])'
|
||||
_TESTS = [{
|
||||
'url': 'https://budem.mave.digital/',
|
||||
'info_dict': {
|
||||
'id': 'budem',
|
||||
'title': 'Все там будем',
|
||||
'description': 'md5:f04ae12a42be0f1d765c5e326b41987a',
|
||||
},
|
||||
'playlist_mincount': 15,
|
||||
}, {
|
||||
'url': 'https://ochenlichnoe.mave.digital/',
|
||||
'info_dict': {
|
||||
'id': 'ochenlichnoe',
|
||||
'title': 'Очень личное',
|
||||
'description': 'md5:ee36a6a52546b91b487fe08c552fdbb2',
|
||||
},
|
||||
'playlist_mincount': 20,
|
||||
}, {
|
||||
'url': 'https://geekcity.mave.digital/',
|
||||
'info_dict': {
|
||||
'id': 'geekcity',
|
||||
'title': 'Мужчины в трико',
|
||||
'description': 'md5:4164d425d60a0d97abdce9d1f6f8e049',
|
||||
},
|
||||
'playlist_mincount': 80,
|
||||
}]
|
||||
_PAGE_SIZE = 50
|
||||
|
||||
def _entries(self, channel_id, channel_meta, page_num):
|
||||
page_data = self._download_json(
|
||||
f'{self._API_BASE_URL}/{channel_id}/episodes', channel_id, query={
|
||||
'view': 'all',
|
||||
'page': page_num + 1,
|
||||
'sort': 'newest',
|
||||
'format': 'all',
|
||||
}, note=f'Downloading page {page_num + 1}')
|
||||
for ep in traverse_obj(page_data, ('episodes', lambda _, v: v['audio'] and v['id'])):
|
||||
yield self._create_entry(channel_id, channel_meta, ep)
|
||||
|
||||
def _real_extract(self, url):
|
||||
channel_id = self._match_id(url)
|
||||
|
||||
channel_meta = self._load_channel_meta(channel_id, channel_id)
|
||||
|
||||
return {
|
||||
'display_id': display_id,
|
||||
'channel_id': channel_id,
|
||||
'channel_url': f'https://{channel_id}.mave.digital/',
|
||||
'vcodec': 'none',
|
||||
'thumbnail': re.sub(r'_\d+(?=\.(?:jpg|png))', '', self._og_search_thumbnail(webpage, default='')) or None,
|
||||
**traverse_obj(data, ('activeEpisodeData', {
|
||||
'url': ('audio', {urljoin(self._API_BASE_URL)}),
|
||||
'id': ('id', {str}),
|
||||
'_type': 'playlist',
|
||||
'id': channel_id,
|
||||
**traverse_obj(channel_meta, {
|
||||
'title': ('title', {str}),
|
||||
'description': ('description', {clean_html}),
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'season_number': ('season', {int_or_none}),
|
||||
'episode_number': ('number', {int_or_none}),
|
||||
'view_count': ('listenings', {int_or_none}),
|
||||
'like_count': ('reactions', lambda _, v: v['type'] == 'like', 'count', {int_or_none}, any),
|
||||
'dislike_count': ('reactions', lambda _, v: v['type'] == 'dislike', 'count', {int_or_none}, any),
|
||||
'age_limit': ('is_explicit', {bool}, {lambda x: 18 if x else None}),
|
||||
'timestamp': ('publish_date', {parse_iso8601}),
|
||||
})),
|
||||
**traverse_obj(data, ('podcast', 'podcast', {
|
||||
'series_id': ('id', {str}),
|
||||
'series': ('title', {str}),
|
||||
'channel': ('title', {str}),
|
||||
'uploader': ('author', {str}),
|
||||
})),
|
||||
'description': ('description', {str}),
|
||||
}),
|
||||
'entries': InAdvancePagedList(
|
||||
functools.partial(self._entries, channel_id, channel_meta),
|
||||
math.ceil(channel_meta['episodes_count'] / self._PAGE_SIZE), self._PAGE_SIZE),
|
||||
}
|
||||
|
||||
@@ -1,14 +1,9 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
float_or_none,
|
||||
format_field,
|
||||
int_or_none,
|
||||
str_or_none,
|
||||
traverse_obj,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class MedalTVIE(InfoExtractor):
|
||||
@@ -30,25 +25,8 @@ class MedalTVIE(InfoExtractor):
|
||||
'view_count': int,
|
||||
'like_count': int,
|
||||
'duration': 13,
|
||||
},
|
||||
}, {
|
||||
'url': 'https://medal.tv/games/cod-cold-war/clips/2mA60jWAGQCBH',
|
||||
'md5': 'fc7a3e4552ae8993c1c4006db46be447',
|
||||
'info_dict': {
|
||||
'id': '2mA60jWAGQCBH',
|
||||
'ext': 'mp4',
|
||||
'title': 'Quad Cold',
|
||||
'description': 'Medal,https://medal.tv/desktop/',
|
||||
'uploader': 'MowgliSB',
|
||||
'timestamp': 1603165266,
|
||||
'upload_date': '20201020',
|
||||
'uploader_id': '10619174',
|
||||
'thumbnail': 'https://cdn.medal.tv/10619174/thumbnail-34934644-720p.jpg?t=1080p&c=202042&missing',
|
||||
'uploader_url': 'https://medal.tv/users/10619174',
|
||||
'comment_count': int,
|
||||
'view_count': int,
|
||||
'like_count': int,
|
||||
'duration': 23,
|
||||
'thumbnail': r're:https://cdn\.medal\.tv/ugcp/content-thumbnail/.*\.jpg',
|
||||
'tags': ['headshot', 'valorant', '4k', 'clutch', 'mornu'],
|
||||
},
|
||||
}, {
|
||||
'url': 'https://medal.tv/games/cod-cold-war/clips/2um24TWdty0NA',
|
||||
@@ -57,12 +35,12 @@ class MedalTVIE(InfoExtractor):
|
||||
'id': '2um24TWdty0NA',
|
||||
'ext': 'mp4',
|
||||
'title': 'u tk me i tk u bigger',
|
||||
'description': 'Medal,https://medal.tv/desktop/',
|
||||
'uploader': 'Mimicc',
|
||||
'description': '',
|
||||
'uploader': 'zahl',
|
||||
'timestamp': 1605580939,
|
||||
'upload_date': '20201117',
|
||||
'uploader_id': '5156321',
|
||||
'thumbnail': 'https://cdn.medal.tv/5156321/thumbnail-36787208-360p.jpg?t=1080p&c=202046&missing',
|
||||
'thumbnail': r're:https://cdn\.medal\.tv/source/.*\.png',
|
||||
'uploader_url': 'https://medal.tv/users/5156321',
|
||||
'comment_count': int,
|
||||
'view_count': int,
|
||||
@@ -70,91 +48,77 @@ class MedalTVIE(InfoExtractor):
|
||||
'duration': 9,
|
||||
},
|
||||
}, {
|
||||
'url': 'https://medal.tv/games/valorant/clips/37rMeFpryCC-9',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
# API requires auth
|
||||
'url': 'https://medal.tv/games/valorant/clips/2WRj40tpY_EU9',
|
||||
'md5': '6c6bb6569777fd8b4ef7b33c09de8dcf',
|
||||
'info_dict': {
|
||||
'id': '2WRj40tpY_EU9',
|
||||
'ext': 'mp4',
|
||||
'title': '1v5 clutch',
|
||||
'description': '',
|
||||
'uploader': 'adny',
|
||||
'uploader_id': '6256941',
|
||||
'uploader_url': 'https://medal.tv/users/6256941',
|
||||
'comment_count': int,
|
||||
'view_count': int,
|
||||
'like_count': int,
|
||||
'duration': 25,
|
||||
'thumbnail': r're:https://cdn\.medal\.tv/source/.*\.jpg',
|
||||
'timestamp': 1612896680,
|
||||
'upload_date': '20210209',
|
||||
},
|
||||
'expected_warnings': ['Video formats are not available through API'],
|
||||
}, {
|
||||
'url': 'https://medal.tv/games/valorant/clips/37rMeFpryCC-9',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id, query={'mobilebypass': 'true'})
|
||||
|
||||
hydration_data = self._search_json(
|
||||
r'<script[^>]*>[^<]*\bhydrationData\s*=', webpage,
|
||||
'next data', video_id, end_pattern='</script>', fatal=False)
|
||||
|
||||
clip = traverse_obj(hydration_data, ('clips', ...), get_all=False)
|
||||
if not clip:
|
||||
raise ExtractorError(
|
||||
'Could not find video information.', video_id=video_id)
|
||||
|
||||
title = clip['contentTitle']
|
||||
|
||||
source_width = int_or_none(clip.get('sourceWidth'))
|
||||
source_height = int_or_none(clip.get('sourceHeight'))
|
||||
|
||||
aspect_ratio = source_width / source_height if source_width and source_height else 16 / 9
|
||||
|
||||
def add_item(container, item_url, height, id_key='format_id', item_id=None):
|
||||
item_id = item_id or '%dp' % height
|
||||
if item_id not in item_url:
|
||||
return
|
||||
container.append({
|
||||
'url': item_url,
|
||||
id_key: item_id,
|
||||
'width': round(aspect_ratio * height),
|
||||
'height': height,
|
||||
})
|
||||
content_data = self._download_json(
|
||||
f'https://medal.tv/api/content/{video_id}', video_id,
|
||||
headers={'Accept': 'application/json'})
|
||||
|
||||
formats = []
|
||||
thumbnails = []
|
||||
for k, v in clip.items():
|
||||
if not (v and isinstance(v, str)):
|
||||
continue
|
||||
mobj = re.match(r'(contentUrl|thumbnail)(?:(\d+)p)?$', k)
|
||||
if not mobj:
|
||||
continue
|
||||
prefix = mobj.group(1)
|
||||
height = int_or_none(mobj.group(2))
|
||||
if prefix == 'contentUrl':
|
||||
add_item(
|
||||
formats, v, height or source_height,
|
||||
item_id=None if height else 'source')
|
||||
elif prefix == 'thumbnail':
|
||||
add_item(thumbnails, v, height, 'id')
|
||||
|
||||
error = clip.get('error')
|
||||
if not formats and error:
|
||||
if error == 404:
|
||||
self.raise_no_formats(
|
||||
'That clip does not exist.',
|
||||
expected=True, video_id=video_id)
|
||||
else:
|
||||
self.raise_no_formats(
|
||||
f'An unknown error occurred ({error}).',
|
||||
video_id=video_id)
|
||||
|
||||
# Necessary because the id of the author is not known in advance.
|
||||
# Won't raise an issue if no profile can be found as this is optional.
|
||||
author = traverse_obj(hydration_data, ('profiles', ...), get_all=False) or {}
|
||||
author_id = str_or_none(author.get('userId'))
|
||||
author_url = format_field(author_id, None, 'https://medal.tv/users/%s')
|
||||
if m3u8_url := url_or_none(content_data.get('contentUrlHls')):
|
||||
formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', m3u8_id='hls'))
|
||||
if http_url := url_or_none(content_data.get('contentUrl')):
|
||||
formats.append({
|
||||
'url': http_url,
|
||||
'format_id': 'http-source',
|
||||
'ext': 'mp4',
|
||||
'quality': 1,
|
||||
})
|
||||
formats = [fmt for fmt in formats if 'video/privacy-protected-guest' not in fmt['url']]
|
||||
if not formats:
|
||||
# Fallback, does not require auth
|
||||
self.report_warning('Video formats are not available through API, falling back to social video URL')
|
||||
urlh = self._request_webpage(
|
||||
f'https://medal.tv/api/content/{video_id}/socialVideoUrl', video_id,
|
||||
note='Checking social video URL')
|
||||
formats.append({
|
||||
'url': urlh.url,
|
||||
'format_id': 'social-video',
|
||||
'ext': 'mp4',
|
||||
'quality': -1,
|
||||
})
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'formats': formats,
|
||||
'thumbnails': thumbnails,
|
||||
'description': clip.get('contentDescription'),
|
||||
'uploader': author.get('displayName'),
|
||||
'timestamp': float_or_none(clip.get('created'), 1000),
|
||||
'uploader_id': author_id,
|
||||
'uploader_url': author_url,
|
||||
'duration': int_or_none(clip.get('videoLengthSeconds')),
|
||||
'view_count': int_or_none(clip.get('views')),
|
||||
'like_count': int_or_none(clip.get('likes')),
|
||||
'comment_count': int_or_none(clip.get('comments')),
|
||||
**traverse_obj(content_data, {
|
||||
'title': ('contentTitle', {str}),
|
||||
'description': ('contentDescription', {str}),
|
||||
'timestamp': ('created', {int_or_none(scale=1000)}),
|
||||
'duration': ('videoLengthSeconds', {int_or_none}),
|
||||
'view_count': ('views', {int_or_none}),
|
||||
'like_count': ('likes', {int_or_none}),
|
||||
'comment_count': ('comments', {int_or_none}),
|
||||
'uploader': ('poster', 'displayName', {str}),
|
||||
'uploader_id': ('poster', 'userId', {str}),
|
||||
'uploader_url': ('poster', 'userId', {str}, filter, {lambda x: x and f'https://medal.tv/users/{x}'}),
|
||||
'tags': ('tags', ..., {str}),
|
||||
'thumbnail': ('thumbnailUrl', {url_or_none}),
|
||||
}),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
extract_attributes,
|
||||
filter_dict,
|
||||
parse_qs,
|
||||
smuggle_url,
|
||||
unsmuggle_url,
|
||||
update_url_query,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class MuxIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:stream\.new/v|player\.mux\.com)/(?P<id>[A-Za-z0-9-]+)'
|
||||
_EMBED_REGEX = [r'<iframe\b[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:stream\.new/v|player\.mux\.com)/(?P<id>[A-Za-z0-9-]+)[^"\']+)']
|
||||
_TESTS = [{
|
||||
'url': 'https://stream.new/v/OCtRWZiZqKvLbnZ32WSEYiGNvHdAmB01j/embed',
|
||||
'info_dict': {
|
||||
'ext': 'mp4',
|
||||
'id': 'OCtRWZiZqKvLbnZ32WSEYiGNvHdAmB01j',
|
||||
'title': 'OCtRWZiZqKvLbnZ32WSEYiGNvHdAmB01j',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://player.mux.com/OCtRWZiZqKvLbnZ32WSEYiGNvHdAmB01j',
|
||||
'info_dict': {
|
||||
'ext': 'mp4',
|
||||
'id': 'OCtRWZiZqKvLbnZ32WSEYiGNvHdAmB01j',
|
||||
'title': 'OCtRWZiZqKvLbnZ32WSEYiGNvHdAmB01j',
|
||||
},
|
||||
}]
|
||||
_WEBPAGE_TESTS = [{
|
||||
# iframe embed
|
||||
'url': 'https://www.redbrickai.com/blog/2025-07-14-FAST-brush',
|
||||
'info_dict': {
|
||||
'ext': 'mp4',
|
||||
'id': 'cXhzAiW1AmsHY01eRbEYFcTEAn0102aGN8sbt8JprP6Dfw',
|
||||
'title': 'cXhzAiW1AmsHY01eRbEYFcTEAn0102aGN8sbt8JprP6Dfw',
|
||||
},
|
||||
}, {
|
||||
# mux-player embed
|
||||
'url': 'https://muxvideo.2coders.com/download/',
|
||||
'info_dict': {
|
||||
'ext': 'mp4',
|
||||
'id': 'JBuasdg35Hw7tYmTe9k68QLPQKixL300YsWHDz5Flit8',
|
||||
'title': 'JBuasdg35Hw7tYmTe9k68QLPQKixL300YsWHDz5Flit8',
|
||||
},
|
||||
}, {
|
||||
# mux-player with title metadata
|
||||
'url': 'https://datastar-todomvc.cross.stream/',
|
||||
'info_dict': {
|
||||
'ext': 'mp4',
|
||||
'id': 'KX01ZSZ8CXv5SVfVwMZKJTcuBcUQmo1ReS9U5JjoHm4k',
|
||||
'title': 'TodoMVC with Datastar Tutorial',
|
||||
},
|
||||
}]
|
||||
|
||||
@classmethod
|
||||
def _extract_embed_urls(cls, url, webpage):
|
||||
yield from super()._extract_embed_urls(url, webpage)
|
||||
for mux_player in re.findall(r'<mux-(?:player|video)\b[^>]*\bplayback-id=[^>]+>', webpage):
|
||||
attrs = extract_attributes(mux_player)
|
||||
playback_id = attrs.get('playback-id')
|
||||
if not playback_id:
|
||||
continue
|
||||
token = attrs.get('playback-token') or traverse_obj(playback_id, ({parse_qs}, 'token', -1))
|
||||
playback_id = playback_id.partition('?')[0]
|
||||
|
||||
embed_url = update_url_query(
|
||||
f'https://player.mux.com/{playback_id}',
|
||||
filter_dict({'playback-token': token}))
|
||||
if title := attrs.get('metadata-video-title'):
|
||||
embed_url = smuggle_url(embed_url, {'title': title})
|
||||
yield embed_url
|
||||
|
||||
def _real_extract(self, url):
|
||||
url, smuggled_data = unsmuggle_url(url, {})
|
||||
video_id = self._match_id(url)
|
||||
|
||||
token = traverse_obj(parse_qs(url), ('playback-token', -1))
|
||||
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||
f'https://stream.mux.com/{video_id}.m3u8', video_id, 'mp4',
|
||||
query=filter_dict({'token': token}))
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': smuggled_data.get('title') or video_id,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
float_or_none,
|
||||
parse_iso8601,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class NascarClassicsIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?classics\.nascar\.com/video/(?P<id>[\w~-]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://classics.nascar.com/video/Ka5qGuxzZ~SIvJii7uAC~wszPshklHN',
|
||||
'md5': '81d712eccffa7169c328281b8cc28f77',
|
||||
'info_dict': {
|
||||
'id': 'Ka5qGuxzZ~SIvJii7uAC~wszPshklHN',
|
||||
'ext': 'mp4',
|
||||
'title': 'Cook Out 400 2023',
|
||||
'thumbnail': 'https://va.aws.nascar.com/IMAGES/CUP_2023_22_RICHMOND_THUMB_NCD.jpg',
|
||||
'timestamp': 1690732800,
|
||||
'upload_date': '20230730',
|
||||
'tags': ['2023', 'race #22', 'richmond', 'chris buescher', 'cup'],
|
||||
'chapters': 'count:18',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://classics.nascar.com/video/UASvPDOwEha~SIvJii7uAC~wszPshklHN',
|
||||
'md5': 'a5e8d6ec6005da3857d25ba2df5e7133',
|
||||
'info_dict': {
|
||||
'id': 'UASvPDOwEha~SIvJii7uAC~wszPshklHN',
|
||||
'ext': 'mp4',
|
||||
'title': 'I Love New York 355 at the Glen 2017',
|
||||
'thumbnail': 'https://va.aws.nascar.com/IMAGES/CUP_2017_22_WATKINSGLEN_THUMB_NCD.jpg',
|
||||
'timestamp': 1501995600,
|
||||
'upload_date': '20170806',
|
||||
'tags': ['watkins glen', 'race #22', '2017', 'martin truex jr.', 'cup'],
|
||||
'chapters': 'count:13',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
content_data = self._search_nextjs_data(
|
||||
webpage, video_id)['props']['pageProps']['contentData']
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'formats': self._extract_m3u8_formats(content_data['input']['src'], video_id, 'mp4'),
|
||||
**traverse_obj(content_data, {
|
||||
'title': ('input', 'name', {str}),
|
||||
'description': ('input', 'description', {str}, filter),
|
||||
'thumbnail': ('input', 'thumbnail', {url_or_none}),
|
||||
'tags': ('input', 'settings', 'tags', ..., {str}),
|
||||
'timestamp': ('input', 'start_time', {parse_iso8601}),
|
||||
'chapters': ('overlay', 'data', 'timelines', 0, 'events', lambda _, v: float(v['timestamp']) is not None, {
|
||||
'start_time': ('timestamp', {float_or_none}),
|
||||
'title': ('name', {str}),
|
||||
}),
|
||||
}),
|
||||
}
|
||||
@@ -63,7 +63,7 @@ class NBCUniversalBaseIE(ThePlatformBaseIE):
|
||||
# formats='mpeg4' will return either a working m3u8 URL or an m3u8 template for non-DRM HLS
|
||||
# formats='m3u+none,mpeg4' may return DRM HLS but w/the "folders" needed for non-DRM template
|
||||
query['formats'] = 'm3u+none,mpeg4'
|
||||
m3u8_url = self._download_nbcu_smil_and_extract_m3u8_url(tp_path, video_id, query)
|
||||
orig_m3u8_url = m3u8_url = self._download_nbcu_smil_and_extract_m3u8_url(tp_path, video_id, query)
|
||||
|
||||
if mobj := re.fullmatch(self._M3U8_RE, m3u8_url):
|
||||
query['formats'] = 'mpeg4'
|
||||
@@ -76,7 +76,17 @@ class NBCUniversalBaseIE(ThePlatformBaseIE):
|
||||
if '/mpeg_cenc' in m3u8_url or '/mpeg_cbcs' in m3u8_url:
|
||||
self.report_drm(video_id)
|
||||
|
||||
return self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||
m3u8_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
|
||||
|
||||
if not formats and m3u8_url != orig_m3u8_url:
|
||||
orig_fmts, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||
orig_m3u8_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
|
||||
formats = [f for f in orig_fmts if not f.get('has_drm')]
|
||||
if orig_fmts and not formats:
|
||||
self.report_drm(video_id)
|
||||
|
||||
return formats, subtitles
|
||||
|
||||
def _extract_nbcu_video(self, url, display_id, old_ie_key=None):
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
|
||||
@@ -478,3 +478,64 @@ class NebulaChannelIE(NebulaBaseIE):
|
||||
playlist_id=collection_slug,
|
||||
playlist_title=channel.get('title'),
|
||||
playlist_description=channel.get('description'))
|
||||
|
||||
|
||||
class NebulaSeasonIE(NebulaBaseIE):
|
||||
IE_NAME = 'nebula:season'
|
||||
_VALID_URL = rf'{_BASE_URL_RE}/(?P<series>[\w-]+)/season/(?P<season_number>[\w-]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://nebula.tv/jetlag/season/15',
|
||||
'info_dict': {
|
||||
'id': 'jetlag_15',
|
||||
'title': 'Tag: All Stars',
|
||||
'description': 'md5:5aa5b8abf3de71756448dc44ffebb674',
|
||||
},
|
||||
'playlist_count': 8,
|
||||
}, {
|
||||
'url': 'https://nebula.tv/jetlag/season/14',
|
||||
'info_dict': {
|
||||
'id': 'jetlag_14',
|
||||
'title': 'Snake',
|
||||
'description': 'md5:6da9040f1c2ac559579738bfb6919d1e',
|
||||
},
|
||||
'playlist_count': 8,
|
||||
}, {
|
||||
'url': 'https://nebula.tv/jetlag/season/13-5',
|
||||
'info_dict': {
|
||||
'id': 'jetlag_13-5',
|
||||
'title': 'Hide + Seek Across NYC',
|
||||
'description': 'md5:5b87bb9acc6dcdff289bb4c71a2ad59f',
|
||||
},
|
||||
'playlist_count': 3,
|
||||
}]
|
||||
|
||||
def _build_url_result(self, item):
|
||||
url = (
|
||||
traverse_obj(item, ('share_url', {url_or_none}))
|
||||
or urljoin('https://nebula.tv/', item.get('app_path'))
|
||||
or f'https://nebula.tv/videos/{item["slug"]}')
|
||||
return self.url_result(
|
||||
smuggle_url(url, {'id': item['id']}),
|
||||
NebulaIE, url_transparent=True,
|
||||
**self._extract_video_metadata(item))
|
||||
|
||||
def _entries(self, data):
|
||||
for episode in traverse_obj(data, ('episodes', lambda _, v: v['video']['id'], 'video')):
|
||||
yield self._build_url_result(episode)
|
||||
for extra in traverse_obj(data, ('extras', ..., 'items', lambda _, v: v['id'])):
|
||||
yield self._build_url_result(extra)
|
||||
for trailer in traverse_obj(data, ('trailers', lambda _, v: v['id'])):
|
||||
yield self._build_url_result(trailer)
|
||||
|
||||
def _real_extract(self, url):
|
||||
series, season_id = self._match_valid_url(url).group('series', 'season_number')
|
||||
playlist_id = f'{series}_{season_id}'
|
||||
data = self._call_api(
|
||||
f'https://content.api.nebula.app/content/{series}/season/{season_id}', playlist_id)
|
||||
|
||||
return self.playlist_result(
|
||||
self._entries(data), playlist_id,
|
||||
**traverse_obj(data, {
|
||||
'title': ('title', {str}),
|
||||
'description': ('description', {str}),
|
||||
}))
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
from .brightcove import BrightcoveNewIE
|
||||
from .common import InfoExtractor
|
||||
from ..utils import parse_iso8601
|
||||
from ..utils.traversal import require, traverse_obj
|
||||
|
||||
|
||||
class NetAppBaseIE(InfoExtractor):
|
||||
_BC_URL = 'https://players.brightcove.net/6255154784001/default_default/index.html?videoId={}'
|
||||
|
||||
@staticmethod
|
||||
def _parse_metadata(item):
|
||||
return traverse_obj(item, {
|
||||
'title': ('name', {str}),
|
||||
'description': ('description', {str}),
|
||||
'timestamp': ('createdAt', {parse_iso8601}),
|
||||
})
|
||||
|
||||
|
||||
class NetAppVideoIE(NetAppBaseIE):
|
||||
_VALID_URL = r'https?://media\.netapp\.com/video-detail/(?P<id>[0-9a-f-]+)'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://media.netapp.com/video-detail/da25fc01-82ad-5284-95bc-26920200a222/seamless-storage-for-modern-kubernetes-deployments',
|
||||
'info_dict': {
|
||||
'id': '1843620950167202073',
|
||||
'ext': 'mp4',
|
||||
'title': 'Seamless storage for modern Kubernetes deployments',
|
||||
'description': 'md5:1ee39e315243fe71fb90af2796037248',
|
||||
'uploader_id': '6255154784001',
|
||||
'duration': 2159.41,
|
||||
'thumbnail': r're:https://house-fastly-signed-us-east-1-prod\.brightcovecdn\.com/image/.*\.jpg',
|
||||
'tags': 'count:15',
|
||||
'timestamp': 1758213949,
|
||||
'upload_date': '20250918',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://media.netapp.com/video-detail/45593e5d-cf1c-5996-978c-c9081906e69f/unleash-ai-innovation-with-your-data-with-the-netapp-platform',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_uuid = self._match_id(url)
|
||||
metadata = self._download_json(
|
||||
f'https://api.media.netapp.com/client/detail/{video_uuid}', video_uuid)
|
||||
|
||||
brightcove_video_id = traverse_obj(metadata, (
|
||||
'sections', lambda _, v: v['type'] == 'Player', 'video', {str}, any, {require('brightcove video id')}))
|
||||
|
||||
video_item = traverse_obj(metadata, ('sections', lambda _, v: v['type'] == 'VideoDetail', any))
|
||||
|
||||
return self.url_result(
|
||||
self._BC_URL.format(brightcove_video_id), BrightcoveNewIE, brightcove_video_id,
|
||||
url_transparent=True, **self._parse_metadata(video_item))
|
||||
|
||||
|
||||
class NetAppCollectionIE(NetAppBaseIE):
|
||||
_VALID_URL = r'https?://media\.netapp\.com/collection/(?P<id>[0-9a-f-]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://media.netapp.com/collection/9820e190-f2a6-47ac-9c0a-98e5e64234a4',
|
||||
'info_dict': {
|
||||
'title': 'Featured sessions',
|
||||
'id': '9820e190-f2a6-47ac-9c0a-98e5e64234a4',
|
||||
},
|
||||
'playlist_count': 4,
|
||||
}]
|
||||
|
||||
def _entries(self, metadata):
|
||||
for item in traverse_obj(metadata, ('items', lambda _, v: v['brightcoveVideoId'])):
|
||||
brightcove_video_id = item['brightcoveVideoId']
|
||||
yield self.url_result(
|
||||
self._BC_URL.format(brightcove_video_id), BrightcoveNewIE, brightcove_video_id,
|
||||
url_transparent=True, **self._parse_metadata(item))
|
||||
|
||||
def _real_extract(self, url):
|
||||
collection_uuid = self._match_id(url)
|
||||
metadata = self._download_json(
|
||||
f'https://api.media.netapp.com/client/collection/{collection_uuid}', collection_uuid)
|
||||
|
||||
return self.playlist_result(self._entries(metadata), collection_uuid, playlist_title=metadata.get('name'))
|
||||
@@ -528,7 +528,7 @@ class NetEaseMusicMvIE(NetEaseMusicBaseIE):
|
||||
class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
|
||||
IE_NAME = 'netease:program'
|
||||
IE_DESC = '网易云音乐 - 电台节目'
|
||||
_VALID_URL = r'https?://music\.163\.com/(?:#/)?program\?id=(?P<id>[0-9]+)'
|
||||
_VALID_URL = r'https?://music\.163\.com/(?:#/)?(?:dj|program)\?id=(?P<id>[0-9]+)'
|
||||
_TESTS = [{
|
||||
'url': 'http://music.163.com/#/program?id=10109055',
|
||||
'info_dict': {
|
||||
@@ -572,6 +572,9 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
|
||||
'params': {
|
||||
'noplaylist': True,
|
||||
},
|
||||
}, {
|
||||
'url': 'https://music.163.com/#/dj?id=3706179315',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
|
||||
@@ -2,84 +2,59 @@ from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
clean_html,
|
||||
int_or_none,
|
||||
js_to_json,
|
||||
parse_iso8601,
|
||||
url_or_none,
|
||||
urljoin,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class NetzkinoIE(InfoExtractor):
|
||||
_WORKING = False
|
||||
_VALID_URL = r'https?://(?:www\.)?netzkino\.de/\#!/[^/]+/(?P<id>[^/]+)'
|
||||
|
||||
_GEO_COUNTRIES = ['DE']
|
||||
_VALID_URL = r'https?://(?:www\.)?netzkino\.de/details/(?P<id>[^/?#]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.netzkino.de/#!/scifikino/rakete-zum-mond',
|
||||
'md5': '92a3f8b76f8d7220acce5377ea5d4873',
|
||||
'url': 'https://www.netzkino.de/details/snow-beast',
|
||||
'md5': '1a4c90fe40d3ccabce163287e45e56dd',
|
||||
'info_dict': {
|
||||
'id': 'rakete-zum-mond',
|
||||
'id': 'snow-beast',
|
||||
'ext': 'mp4',
|
||||
'title': 'Rakete zum Mond \u2013 Jules Verne',
|
||||
'description': 'md5:f0a8024479618ddbfa450ff48ffa6c60',
|
||||
'upload_date': '20120813',
|
||||
'thumbnail': r're:https?://.*\.jpg$',
|
||||
'timestamp': 1344858571,
|
||||
'title': 'Snow Beast',
|
||||
'age_limit': 12,
|
||||
},
|
||||
'params': {
|
||||
'skip_download': 'Download only works from Germany',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.netzkino.de/#!/filme/dr-jekyll-mrs-hyde-2',
|
||||
'md5': 'c7728b2dadd04ff6727814847a51ef03',
|
||||
'info_dict': {
|
||||
'id': 'dr-jekyll-mrs-hyde-2',
|
||||
'ext': 'mp4',
|
||||
'title': 'Dr. Jekyll & Mrs. Hyde 2',
|
||||
'description': 'md5:c2e9626ebd02de0a794b95407045d186',
|
||||
'upload_date': '20190130',
|
||||
'thumbnail': r're:https?://.*\.jpg$',
|
||||
'timestamp': 1548849437,
|
||||
'age_limit': 18,
|
||||
},
|
||||
'params': {
|
||||
'skip_download': 'Download only works from Germany',
|
||||
'alt_title': 'Snow Beast',
|
||||
'cast': 'count:3',
|
||||
'categories': 'count:7',
|
||||
'creators': 'count:2',
|
||||
'description': 'md5:e604a954a7f827a80e96a3a97d48b269',
|
||||
'location': 'US',
|
||||
'release_year': 2011,
|
||||
'thumbnail': r're:https?://.+\.jpg',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = self._match_valid_url(url)
|
||||
video_id = mobj.group('id')
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
next_js_data = self._search_nextjs_data(webpage, video_id)
|
||||
|
||||
api_url = f'https://api.netzkino.de.simplecache.net/capi-2.0a/movies/{video_id}.json?d=www'
|
||||
info = self._download_json(api_url, video_id)
|
||||
custom_fields = info['custom_fields']
|
||||
|
||||
production_js = self._download_webpage(
|
||||
'http://www.netzkino.de/beta/dist/production.min.js', video_id,
|
||||
note='Downloading player code')
|
||||
avo_js = self._search_regex(
|
||||
r'var urlTemplate=(\{.*?"\})',
|
||||
production_js, 'URL templates')
|
||||
templates = self._parse_json(
|
||||
avo_js, video_id, transform_source=js_to_json)
|
||||
|
||||
suffix = {
|
||||
'hds': '.mp4/manifest.f4m',
|
||||
'hls': '.mp4/master.m3u8',
|
||||
'pmd': '.mp4',
|
||||
}
|
||||
film_fn = custom_fields['Streaming'][0]
|
||||
formats = [{
|
||||
'format_id': key,
|
||||
'ext': 'mp4',
|
||||
'url': tpl.replace('{}', film_fn) + suffix[key],
|
||||
} for key, tpl in templates.items()]
|
||||
query = traverse_obj(next_js_data, (
|
||||
'props', '__dehydratedState', 'queries', ..., 'state',
|
||||
'data', 'data', lambda _, v: v['__typename'] == 'CmsMovie', any))
|
||||
if 'DRM' in traverse_obj(query, ('licenses', 'nodes', ..., 'properties', {str})):
|
||||
self.report_drm(video_id)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
'title': info['title'],
|
||||
'age_limit': int_or_none(custom_fields.get('FSK')[0]),
|
||||
'timestamp': parse_iso8601(info.get('date'), delimiter=' '),
|
||||
'description': clean_html(info.get('content')),
|
||||
'thumbnail': info.get('thumbnail'),
|
||||
**traverse_obj(query, {
|
||||
'title': ('originalTitle', {clean_html}),
|
||||
'age_limit': ('fskRating', {int_or_none}),
|
||||
'alt_title': ('originalTitle', {clean_html}, filter),
|
||||
'cast': ('cast', 'nodes', ..., 'person', 'name', {clean_html}, filter),
|
||||
'creators': (('directors', 'writers'), 'nodes', ..., 'person', 'name', {clean_html}, filter),
|
||||
'categories': ('categories', 'nodes', ..., 'category', 'title', {clean_html}, filter),
|
||||
'description': ('longSynopsis', {clean_html}, filter),
|
||||
'duration': ('runtimeInSeconds', {int_or_none}),
|
||||
'location': ('productionCountry', {clean_html}, filter),
|
||||
'release_year': ('productionYear', {int_or_none}),
|
||||
'thumbnail': ('coverImage', 'masterUrl', {url_or_none}),
|
||||
'url': ('videoSource', 'pmdUrl', {urljoin('https://pmd.netzkino-seite.netzkino.de/')}),
|
||||
}),
|
||||
}
|
||||
|
||||
@@ -1,238 +0,0 @@
|
||||
import urllib.parse
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
clean_html,
|
||||
get_element_by_class,
|
||||
int_or_none,
|
||||
parse_iso8601,
|
||||
remove_start,
|
||||
unified_timestamp,
|
||||
)
|
||||
|
||||
|
||||
class NextMediaIE(InfoExtractor):
|
||||
IE_DESC = '蘋果日報'
|
||||
_VALID_URL = r'https?://hk\.apple\.nextmedia\.com/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'http://hk.apple.nextmedia.com/realtime/news/20141108/53109199',
|
||||
'md5': 'dff9fad7009311c421176d1ac90bfe4f',
|
||||
'info_dict': {
|
||||
'id': '53109199',
|
||||
'ext': 'mp4',
|
||||
'title': '【佔領金鐘】50外國領事議員撐場 讚學生勇敢香港有希望',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'description': 'md5:28222b9912b6665a21011b034c70fcc7',
|
||||
'timestamp': 1415456273,
|
||||
'upload_date': '20141108',
|
||||
},
|
||||
}]
|
||||
|
||||
_URL_PATTERN = r'\{ url: \'(.+)\' \}'
|
||||
|
||||
def _real_extract(self, url):
|
||||
news_id = self._match_id(url)
|
||||
page = self._download_webpage(url, news_id)
|
||||
return self._extract_from_nextmedia_page(news_id, url, page)
|
||||
|
||||
def _extract_from_nextmedia_page(self, news_id, url, page):
|
||||
redirection_url = self._search_regex(
|
||||
r'window\.location\.href\s*=\s*([\'"])(?P<url>(?!\1).+)\1',
|
||||
page, 'redirection URL', default=None, group='url')
|
||||
if redirection_url:
|
||||
return self.url_result(urllib.parse.urljoin(url, redirection_url))
|
||||
|
||||
title = self._fetch_title(page)
|
||||
video_url = self._search_regex(self._URL_PATTERN, page, 'video url')
|
||||
|
||||
attrs = {
|
||||
'id': news_id,
|
||||
'title': title,
|
||||
'url': video_url, # ext can be inferred from url
|
||||
'thumbnail': self._fetch_thumbnail(page),
|
||||
'description': self._fetch_description(page),
|
||||
}
|
||||
|
||||
timestamp = self._fetch_timestamp(page)
|
||||
if timestamp:
|
||||
attrs['timestamp'] = timestamp
|
||||
else:
|
||||
attrs['upload_date'] = self._fetch_upload_date(url)
|
||||
|
||||
return attrs
|
||||
|
||||
def _fetch_title(self, page):
|
||||
return self._og_search_title(page)
|
||||
|
||||
def _fetch_thumbnail(self, page):
|
||||
return self._og_search_thumbnail(page)
|
||||
|
||||
def _fetch_timestamp(self, page):
|
||||
date_created = self._search_regex('"dateCreated":"([^"]+)"', page, 'created time')
|
||||
return parse_iso8601(date_created)
|
||||
|
||||
def _fetch_upload_date(self, url):
|
||||
return self._search_regex(self._VALID_URL, url, 'upload date', group='date')
|
||||
|
||||
def _fetch_description(self, page):
|
||||
return self._og_search_property('description', page)
|
||||
|
||||
|
||||
class NextMediaActionNewsIE(NextMediaIE): # XXX: Do not subclass from concrete IE
|
||||
IE_DESC = '蘋果日報 - 動新聞'
|
||||
_VALID_URL = r'https?://hk\.dv\.nextmedia\.com/actionnews/[^/]+/(?P<date>\d+)/(?P<id>\d+)/\d+'
|
||||
_TESTS = [{
|
||||
'url': 'http://hk.dv.nextmedia.com/actionnews/hit/20150121/19009428/20061460',
|
||||
'md5': '05fce8ffeed7a5e00665d4b7cf0f9201',
|
||||
'info_dict': {
|
||||
'id': '19009428',
|
||||
'ext': 'mp4',
|
||||
'title': '【壹週刊】細10年男友偷食 50歲邵美琪再失戀',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'description': 'md5:cd802fad1f40fd9ea178c1e2af02d659',
|
||||
'timestamp': 1421791200,
|
||||
'upload_date': '20150120',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
news_id = self._match_id(url)
|
||||
actionnews_page = self._download_webpage(url, news_id)
|
||||
article_url = self._og_search_url(actionnews_page)
|
||||
article_page = self._download_webpage(article_url, news_id)
|
||||
return self._extract_from_nextmedia_page(news_id, url, article_page)
|
||||
|
||||
|
||||
class AppleDailyIE(NextMediaIE): # XXX: Do not subclass from concrete IE
|
||||
IE_DESC = '臺灣蘋果日報'
|
||||
_VALID_URL = r'https?://(www|ent)\.appledaily\.com\.tw/[^/]+/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
|
||||
_TESTS = [{
|
||||
'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
|
||||
'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
|
||||
'info_dict': {
|
||||
'id': '36354694',
|
||||
'ext': 'mp4',
|
||||
'title': '周亭羽走過摩鐵陰霾2男陪吃 九把刀孤寒看醫生',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'description': 'md5:2acd430e59956dc47cd7f67cb3c003f4',
|
||||
'upload_date': '20150128',
|
||||
},
|
||||
}, {
|
||||
'url': 'http://www.appledaily.com.tw/realtimenews/article/strange/20150128/550549/%E4%B8%8D%E6%BB%BF%E8%A2%AB%E8%B8%A9%E8%85%B3%E3%80%80%E5%B1%B1%E6%9D%B1%E5%85%A9%E5%A4%A7%E5%AA%BD%E4%B8%80%E8%B7%AF%E6%89%93%E4%B8%8B%E8%BB%8A',
|
||||
'md5': '86b4e9132d158279c7883822d94ccc49',
|
||||
'info_dict': {
|
||||
'id': '550549',
|
||||
'ext': 'mp4',
|
||||
'title': '不滿被踩腳 山東兩大媽一路打下車',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'description': 'md5:175b4260c1d7c085993474217e4ab1b4',
|
||||
'upload_date': '20150128',
|
||||
},
|
||||
}, {
|
||||
'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003671',
|
||||
'md5': '03df296d95dedc2d5886debbb80cb43f',
|
||||
'info_dict': {
|
||||
'id': '5003671',
|
||||
'ext': 'mp4',
|
||||
'title': '20正妹熱舞 《刀龍傳說Online》火辣上市',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'description': 'md5:23c0aac567dc08c9c16a3161a2c2e3cd',
|
||||
'upload_date': '20150128',
|
||||
},
|
||||
'skip': 'redirect to http://www.appledaily.com.tw/animation/',
|
||||
}, {
|
||||
# No thumbnail
|
||||
'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003673/',
|
||||
'md5': 'b06182cd386ea7bc6115ec7ff0f72aeb',
|
||||
'info_dict': {
|
||||
'id': '5003673',
|
||||
'ext': 'mp4',
|
||||
'title': '半夜尿尿 好像會看到___',
|
||||
'description': 'md5:61d2da7fe117fede148706cdb85ac066',
|
||||
'upload_date': '20150128',
|
||||
},
|
||||
'expected_warnings': [
|
||||
'video thumbnail',
|
||||
],
|
||||
'skip': 'redirect to http://www.appledaily.com.tw/animation/',
|
||||
}, {
|
||||
'url': 'http://www.appledaily.com.tw/appledaily/article/supplement/20140417/35770334/',
|
||||
'md5': 'eaa20e6b9df418c912d7f5dec2ba734d',
|
||||
'info_dict': {
|
||||
'id': '35770334',
|
||||
'ext': 'mp4',
|
||||
'title': '咖啡占卜測 XU裝熟指數',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'description': 'md5:7b859991a6a4fedbdf3dd3b66545c748',
|
||||
'upload_date': '20140417',
|
||||
},
|
||||
}, {
|
||||
'url': 'http://www.appledaily.com.tw/actionnews/appledaily/7/20161003/960588/',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
# Redirected from http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694
|
||||
'url': 'http://ent.appledaily.com.tw/section/article/headline/20150128/36354694',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
_URL_PATTERN = r'\{url: \'(.+)\'\}'
|
||||
|
||||
def _fetch_title(self, page):
|
||||
return (self._html_search_regex(r'<h1 id="h1">([^<>]+)</h1>', page, 'news title', default=None)
|
||||
or self._html_search_meta('description', page, 'news title'))
|
||||
|
||||
def _fetch_thumbnail(self, page):
|
||||
return self._html_search_regex(r"setInitialImage\(\'([^']+)'\)", page, 'video thumbnail', fatal=False)
|
||||
|
||||
def _fetch_timestamp(self, page):
|
||||
return None
|
||||
|
||||
def _fetch_description(self, page):
|
||||
return self._html_search_meta('description', page, 'news description')
|
||||
|
||||
|
||||
class NextTVIE(InfoExtractor):
|
||||
_WORKING = False
|
||||
_ENABLED = None # XXX: pass through to GenericIE
|
||||
IE_DESC = '壹電視'
|
||||
_VALID_URL = r'https?://(?:www\.)?nexttv\.com\.tw/(?:[^/]+/)+(?P<id>\d+)'
|
||||
|
||||
_TEST = {
|
||||
'url': 'http://www.nexttv.com.tw/news/realtime/politics/11779671',
|
||||
'info_dict': {
|
||||
'id': '11779671',
|
||||
'ext': 'mp4',
|
||||
'title': '「超收稅」近4千億! 藍議員籲發消費券',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'timestamp': 1484825400,
|
||||
'upload_date': '20170119',
|
||||
'view_count': int,
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
title = self._html_search_regex(
|
||||
r'<h1[^>]*>([^<]+)</h1>', webpage, 'title')
|
||||
|
||||
data = self._hidden_inputs(webpage)
|
||||
|
||||
video_url = data['ntt-vod-src-detailview']
|
||||
|
||||
date_str = get_element_by_class('date', webpage)
|
||||
timestamp = unified_timestamp(date_str + '+0800') if date_str else None
|
||||
|
||||
view_count = int_or_none(remove_start(
|
||||
clean_html(get_element_by_class('click', webpage)), '點閱:'))
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'url': video_url,
|
||||
'thumbnail': data.get('ntt-vod-img-src'),
|
||||
'timestamp': timestamp,
|
||||
'view_count': view_count,
|
||||
}
|
||||
@@ -23,96 +23,38 @@ from ..utils import (
|
||||
|
||||
|
||||
class NhkBaseIE(InfoExtractor):
|
||||
_API_URL_TEMPLATE = 'https://nwapi.nhk.jp/nhkworld/%sod%slist/v7b/%s/%s/%s/all%s.json'
|
||||
_API_URL_TEMPLATE = 'https://api.nhkworld.jp/showsapi/v1/{lang}/{content_format}_{page_type}/{m_id}{extra_page}'
|
||||
_BASE_URL_REGEX = r'https?://www3\.nhk\.or\.jp/nhkworld/(?P<lang>[a-z]{2})/'
|
||||
|
||||
def _call_api(self, m_id, lang, is_video, is_episode, is_clip):
|
||||
content_format = 'video' if is_video else 'audio'
|
||||
content_type = 'clips' if is_clip else 'episodes'
|
||||
if not is_episode:
|
||||
extra_page = f'/{content_format}_{content_type}'
|
||||
page_type = 'programs'
|
||||
else:
|
||||
extra_page = ''
|
||||
page_type = content_type
|
||||
|
||||
return self._download_json(
|
||||
self._API_URL_TEMPLATE % (
|
||||
'v' if is_video else 'r',
|
||||
'clip' if is_clip else 'esd',
|
||||
'episode' if is_episode else 'program',
|
||||
m_id, lang, '/all' if is_video else ''),
|
||||
m_id, query={'apikey': 'EJfK8jdS57GqlupFgAfAAwr573q01y6k'})['data']['episodes'] or []
|
||||
|
||||
def _get_api_info(self, refresh=True):
|
||||
if not refresh:
|
||||
return self.cache.load('nhk', 'api_info')
|
||||
|
||||
self.cache.store('nhk', 'api_info', {})
|
||||
movie_player_js = self._download_webpage(
|
||||
'https://movie-a.nhk.or.jp/world/player/js/movie-player.js', None,
|
||||
note='Downloading stream API information')
|
||||
api_info = {
|
||||
'url': self._search_regex(
|
||||
r'prod:[^;]+\bapiUrl:\s*[\'"]([^\'"]+)[\'"]', movie_player_js, None, 'stream API url'),
|
||||
'token': self._search_regex(
|
||||
r'prod:[^;]+\btoken:\s*[\'"]([^\'"]+)[\'"]', movie_player_js, None, 'stream API token'),
|
||||
}
|
||||
self.cache.store('nhk', 'api_info', api_info)
|
||||
return api_info
|
||||
|
||||
def _extract_stream_info(self, vod_id):
|
||||
for refresh in (False, True):
|
||||
api_info = self._get_api_info(refresh)
|
||||
if not api_info:
|
||||
continue
|
||||
|
||||
api_url = api_info.pop('url')
|
||||
meta = traverse_obj(
|
||||
self._download_json(
|
||||
api_url, vod_id, 'Downloading stream url info', fatal=False, query={
|
||||
**api_info,
|
||||
'type': 'json',
|
||||
'optional_id': vod_id,
|
||||
'active_flg': 1,
|
||||
}), ('meta', 0))
|
||||
stream_url = traverse_obj(
|
||||
meta, ('movie_url', ('mb_auto', 'auto_sp', 'auto_pc'), {url_or_none}), get_all=False)
|
||||
|
||||
if stream_url:
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(stream_url, vod_id)
|
||||
return {
|
||||
**traverse_obj(meta, {
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'timestamp': ('publication_date', {unified_timestamp}),
|
||||
'release_timestamp': ('insert_date', {unified_timestamp}),
|
||||
'modified_timestamp': ('update_date', {unified_timestamp}),
|
||||
}),
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
}
|
||||
raise ExtractorError('Unable to extract stream url')
|
||||
self._API_URL_TEMPLATE.format(
|
||||
lang=lang, content_format=content_format, page_type=page_type,
|
||||
m_id=m_id, extra_page=extra_page),
|
||||
join_nonempty(m_id, lang))
|
||||
|
||||
def _extract_episode_info(self, url, episode=None):
|
||||
fetch_episode = episode is None
|
||||
lang, m_type, episode_id = NhkVodIE._match_valid_url(url).group('lang', 'type', 'id')
|
||||
is_video = m_type != 'audio'
|
||||
|
||||
if is_video:
|
||||
episode_id = episode_id[:4] + '-' + episode_id[4:]
|
||||
|
||||
if fetch_episode:
|
||||
episode = self._call_api(
|
||||
episode_id, lang, is_video, True, episode_id[:4] == '9999')[0]
|
||||
episode_id, lang, is_video, is_episode=True, is_clip=episode_id[:4] == '9999')
|
||||
|
||||
def get_clean_field(key):
|
||||
return clean_html(episode.get(key + '_clean') or episode.get(key))
|
||||
video_id = join_nonempty('id', 'lang', from_dict=episode)
|
||||
|
||||
title = get_clean_field('sub_title')
|
||||
series = get_clean_field('title')
|
||||
|
||||
thumbnails = []
|
||||
for s, w, h in [('', 640, 360), ('_l', 1280, 720)]:
|
||||
img_path = episode.get('image' + s)
|
||||
if not img_path:
|
||||
continue
|
||||
thumbnails.append({
|
||||
'id': f'{h}p',
|
||||
'height': h,
|
||||
'width': w,
|
||||
'url': 'https://www3.nhk.or.jp' + img_path,
|
||||
})
|
||||
title = episode.get('title')
|
||||
series = traverse_obj(episode, (('video_program', 'audio_program'), any, 'title'))
|
||||
|
||||
episode_name = title
|
||||
if series and title:
|
||||
@@ -125,37 +67,52 @@ class NhkBaseIE(InfoExtractor):
|
||||
episode_name = None
|
||||
|
||||
info = {
|
||||
'id': episode_id + '-' + lang,
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'description': get_clean_field('description'),
|
||||
'thumbnails': thumbnails,
|
||||
'series': series,
|
||||
'episode': episode_name,
|
||||
**traverse_obj(episode, {
|
||||
'description': ('description', {str}),
|
||||
'release_timestamp': ('first_broadcasted_at', {unified_timestamp}),
|
||||
'categories': ('categories', ..., 'name', {str}),
|
||||
'tags': ('tags', ..., 'name', {str}),
|
||||
'thumbnails': ('images', lambda _, v: v['url'], {
|
||||
'url': ('url', {urljoin(url)}),
|
||||
'width': ('width', {int_or_none}),
|
||||
'height': ('height', {int_or_none}),
|
||||
}),
|
||||
'webpage_url': ('url', {urljoin(url)}),
|
||||
}),
|
||||
'extractor_key': NhkVodIE.ie_key(),
|
||||
'extractor': NhkVodIE.IE_NAME,
|
||||
}
|
||||
|
||||
if is_video:
|
||||
vod_id = episode['vod_id']
|
||||
info.update({
|
||||
**self._extract_stream_info(vod_id),
|
||||
'id': vod_id,
|
||||
})
|
||||
|
||||
# XXX: We are assuming that 'video' and 'audio' are mutually exclusive
|
||||
stream_info = traverse_obj(episode, (('video', 'audio'), {dict}, any)) or {}
|
||||
if not stream_info.get('url'):
|
||||
self.raise_no_formats('Stream not found; it has most likely expired', expected=True)
|
||||
else:
|
||||
if fetch_episode:
|
||||
stream_url = stream_info['url']
|
||||
if is_video:
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(stream_url, video_id)
|
||||
info.update({
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
**traverse_obj(stream_info, ({
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'timestamp': ('published_at', {unified_timestamp}),
|
||||
})),
|
||||
})
|
||||
else:
|
||||
# From https://www3.nhk.or.jp/nhkworld/common/player/radio/inline/rod.html
|
||||
audio_path = remove_end(episode['audio']['audio'], '.m4a')
|
||||
audio_path = remove_end(stream_url, '.m4a')
|
||||
info['formats'] = self._extract_m3u8_formats(
|
||||
f'{urljoin("https://vod-stream.nhk.jp", audio_path)}/index.m3u8',
|
||||
episode_id, 'm4a', entry_protocol='m3u8_native',
|
||||
m3u8_id='hls', fatal=False)
|
||||
for f in info['formats']:
|
||||
f['language'] = lang
|
||||
else:
|
||||
info.update({
|
||||
'_type': 'url_transparent',
|
||||
'ie_key': NhkVodIE.ie_key(),
|
||||
'url': url,
|
||||
})
|
||||
|
||||
return info
|
||||
|
||||
|
||||
@@ -168,29 +125,29 @@ class NhkVodIE(NhkBaseIE):
|
||||
# Content available only for a limited period of time. Visit
|
||||
# https://www3.nhk.or.jp/nhkworld/en/ondemand/ for working samples.
|
||||
_TESTS = [{
|
||||
'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2049126/',
|
||||
'url': 'https://www3.nhk.or.jp/nhkworld/en/shows/2049165/',
|
||||
'info_dict': {
|
||||
'id': 'nw_vod_v_en_2049_126_20230413233000_01_1681398302',
|
||||
'id': '2049165-en',
|
||||
'ext': 'mp4',
|
||||
'title': 'Japan Railway Journal - The Tohoku Shinkansen: Full Speed Ahead',
|
||||
'description': 'md5:49f7c5b206e03868a2fdf0d0814b92f6',
|
||||
'title': 'Japan Railway Journal - Choshi Electric Railway: Fighting to Get Back on Track',
|
||||
'description': 'md5:ab57df2fca7f04245148c2e787bb203d',
|
||||
'thumbnail': r're:https://.+/.+\.jpg',
|
||||
'episode': 'The Tohoku Shinkansen: Full Speed Ahead',
|
||||
'episode': 'Choshi Electric Railway: Fighting to Get Back on Track',
|
||||
'series': 'Japan Railway Journal',
|
||||
'modified_timestamp': 1707217907,
|
||||
'timestamp': 1681428600,
|
||||
'release_timestamp': 1693883728,
|
||||
'duration': 1679,
|
||||
'upload_date': '20230413',
|
||||
'modified_date': '20240206',
|
||||
'release_date': '20230905',
|
||||
'duration': 1680,
|
||||
'categories': ['Biz & Tech'],
|
||||
'tags': ['Akita', 'Chiba', 'Trains', 'Transcript', 'All (Japan Navigator)'],
|
||||
'timestamp': 1759055880,
|
||||
'upload_date': '20250928',
|
||||
'release_timestamp': 1758810600,
|
||||
'release_date': '20250925',
|
||||
},
|
||||
}, {
|
||||
# video clip
|
||||
'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999011/',
|
||||
'md5': '153c3016dfd252ba09726588149cf0e7',
|
||||
'info_dict': {
|
||||
'id': 'lpZXIwaDE6_Z-976CPsFdxyICyWUzlT5',
|
||||
'id': '9999011-en',
|
||||
'ext': 'mp4',
|
||||
'title': 'Dining with the Chef - Chef Saito\'s Family recipe: MENCHI-KATSU',
|
||||
'description': 'md5:5aee4a9f9d81c26281862382103b0ea5',
|
||||
@@ -198,24 +155,23 @@ class NhkVodIE(NhkBaseIE):
|
||||
'series': 'Dining with the Chef',
|
||||
'episode': 'Chef Saito\'s Family recipe: MENCHI-KATSU',
|
||||
'duration': 148,
|
||||
'upload_date': '20190816',
|
||||
'release_date': '20230902',
|
||||
'release_timestamp': 1693619292,
|
||||
'modified_timestamp': 1707217907,
|
||||
'modified_date': '20240206',
|
||||
'timestamp': 1565997540,
|
||||
'categories': ['Food'],
|
||||
'tags': ['Washoku'],
|
||||
'timestamp': 1548212400,
|
||||
'upload_date': '20190123',
|
||||
},
|
||||
}, {
|
||||
# radio
|
||||
'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/audio/livinginjapan-20231001-1/',
|
||||
'url': 'https://www3.nhk.or.jp/nhkworld/en/shows/audio/livinginjapan-20240901-1/',
|
||||
'info_dict': {
|
||||
'id': 'livinginjapan-20231001-1-en',
|
||||
'id': 'livinginjapan-20240901-1-en',
|
||||
'ext': 'm4a',
|
||||
'title': 'Living in Japan - Tips for Travelers to Japan / Ramen Vending Machines',
|
||||
'title': 'Living in Japan - Weekend Hiking / Self-protection from crime',
|
||||
'series': 'Living in Japan',
|
||||
'description': 'md5:0a0e2077d8f07a03071e990a6f51bfab',
|
||||
'description': 'md5:4d0e14ab73bdbfedb60a53b093954ed6',
|
||||
'thumbnail': r're:https://.+/.+\.jpg',
|
||||
'episode': 'Tips for Travelers to Japan / Ramen Vending Machines',
|
||||
'episode': 'Weekend Hiking / Self-protection from crime',
|
||||
'categories': ['Interactive'],
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2015173/',
|
||||
@@ -256,96 +212,51 @@ class NhkVodIE(NhkBaseIE):
|
||||
},
|
||||
'skip': 'expires 2023-10-15',
|
||||
}, {
|
||||
# a one-off (single-episode series). title from the api is just '<p></p>'
|
||||
'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/3004952/',
|
||||
# a one-off (single-episode series). title from the api is just null
|
||||
'url': 'https://www3.nhk.or.jp/nhkworld/en/shows/3026036/',
|
||||
'info_dict': {
|
||||
'id': 'nw_vod_v_en_3004_952_20230723091000_01_1690074552',
|
||||
'id': '3026036-en',
|
||||
'ext': 'mp4',
|
||||
'title': 'Barakan Discovers - AMAMI OSHIMA: Isson\'s Treasure Isla',
|
||||
'description': 'md5:5db620c46a0698451cc59add8816b797',
|
||||
'thumbnail': r're:https://.+/.+\.jpg',
|
||||
'release_date': '20230905',
|
||||
'timestamp': 1690103400,
|
||||
'duration': 2939,
|
||||
'release_timestamp': 1693898699,
|
||||
'upload_date': '20230723',
|
||||
'modified_timestamp': 1707217907,
|
||||
'modified_date': '20240206',
|
||||
'episode': 'AMAMI OSHIMA: Isson\'s Treasure Isla',
|
||||
'series': 'Barakan Discovers',
|
||||
'title': 'STATELESS: The Japanese Left Behind in the Philippines',
|
||||
'description': 'md5:9a2fd51cdfa9f52baae28569e0053786',
|
||||
'duration': 2955,
|
||||
'thumbnail': 'https://www3.nhk.or.jp/nhkworld/en/shows/3026036/images/wide_l_QPtWpt4lzVhm3NzPAMIIF35MCg4CdNwcikPaTS5Q.jpg',
|
||||
'categories': ['Documentary', 'Culture & Lifestyle'],
|
||||
'tags': ['Transcript', 'Documentary 360', 'The Pursuit of PEACE'],
|
||||
'timestamp': 1758931800,
|
||||
'upload_date': '20250927',
|
||||
'release_timestamp': 1758931800,
|
||||
'release_date': '20250927',
|
||||
},
|
||||
}, {
|
||||
# /ondemand/video/ url with alphabetical character in 5th position of id
|
||||
'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999a07/',
|
||||
'info_dict': {
|
||||
'id': 'nw_c_en_9999-a07',
|
||||
'id': '9999a07-en',
|
||||
'ext': 'mp4',
|
||||
'episode': 'Mini-Dramas on SDGs: Ep 1 Close the Gender Gap [Director\'s Cut]',
|
||||
'series': 'Mini-Dramas on SDGs',
|
||||
'modified_date': '20240206',
|
||||
'title': 'Mini-Dramas on SDGs - Mini-Dramas on SDGs: Ep 1 Close the Gender Gap [Director\'s Cut]',
|
||||
'description': 'md5:3f9dcb4db22fceb675d90448a040d3f6',
|
||||
'timestamp': 1621962360,
|
||||
'duration': 189,
|
||||
'release_date': '20230903',
|
||||
'modified_timestamp': 1707217907,
|
||||
'timestamp': 1621911600,
|
||||
'duration': 190,
|
||||
'upload_date': '20210525',
|
||||
'thumbnail': r're:https://.+/.+\.jpg',
|
||||
'release_timestamp': 1693713487,
|
||||
'categories': ['Current Affairs', 'Entertainment'],
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999d17/',
|
||||
'info_dict': {
|
||||
'id': 'nw_c_en_9999-d17',
|
||||
'id': '9999d17-en',
|
||||
'ext': 'mp4',
|
||||
'title': 'Flowers of snow blossom - The 72 Pentads of Yamato',
|
||||
'description': 'Today’s focus: Snow',
|
||||
'release_timestamp': 1693792402,
|
||||
'release_date': '20230904',
|
||||
'upload_date': '20220128',
|
||||
'timestamp': 1643370960,
|
||||
'thumbnail': r're:https://.+/.+\.jpg',
|
||||
'duration': 136,
|
||||
'series': '',
|
||||
'modified_date': '20240206',
|
||||
'modified_timestamp': 1707217907,
|
||||
},
|
||||
}, {
|
||||
# new /shows/ url format
|
||||
'url': 'https://www3.nhk.or.jp/nhkworld/en/shows/2032307/',
|
||||
'info_dict': {
|
||||
'id': 'nw_vod_v_en_2032_307_20240321113000_01_1710990282',
|
||||
'ext': 'mp4',
|
||||
'title': 'Japanology Plus - 20th Anniversary Special Part 1',
|
||||
'description': 'md5:817d41fc8e54339ad2a916161ea24faf',
|
||||
'episode': '20th Anniversary Special Part 1',
|
||||
'series': 'Japanology Plus',
|
||||
'thumbnail': r're:https://.+/.+\.jpg',
|
||||
'duration': 1680,
|
||||
'timestamp': 1711020600,
|
||||
'upload_date': '20240321',
|
||||
'release_timestamp': 1711022683,
|
||||
'release_date': '20240321',
|
||||
'modified_timestamp': 1711031012,
|
||||
'modified_date': '20240321',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www3.nhk.or.jp/nhkworld/en/shows/3020025/',
|
||||
'info_dict': {
|
||||
'id': 'nw_vod_v_en_3020_025_20230325144000_01_1679723944',
|
||||
'ext': 'mp4',
|
||||
'title': '100 Ideas to Save the World - Working Styles Evolve',
|
||||
'description': 'md5:9e6c7778eaaf4f7b4af83569649f84d9',
|
||||
'episode': 'Working Styles Evolve',
|
||||
'series': '100 Ideas to Save the World',
|
||||
'thumbnail': r're:https://.+/.+\.jpg',
|
||||
'duration': 899,
|
||||
'upload_date': '20230325',
|
||||
'timestamp': 1679755200,
|
||||
'release_date': '20230905',
|
||||
'release_timestamp': 1693880540,
|
||||
'modified_date': '20240206',
|
||||
'modified_timestamp': 1707217907,
|
||||
'categories': ['Culture & Lifestyle', 'Science & Nature'],
|
||||
'tags': ['Nara', 'Temples & Shrines', 'Winter', 'Snow'],
|
||||
'timestamp': 1643339040,
|
||||
'upload_date': '20220128',
|
||||
},
|
||||
}, {
|
||||
# new /shows/audio/ url format
|
||||
@@ -373,6 +284,7 @@ class NhkVodProgramIE(NhkBaseIE):
|
||||
'id': 'sumo',
|
||||
'title': 'GRAND SUMO Highlights',
|
||||
'description': 'md5:fc20d02dc6ce85e4b72e0273aa52fdbf',
|
||||
'series': 'GRAND SUMO Highlights',
|
||||
},
|
||||
'playlist_mincount': 1,
|
||||
}, {
|
||||
@@ -381,6 +293,7 @@ class NhkVodProgramIE(NhkBaseIE):
|
||||
'id': 'japanrailway',
|
||||
'title': 'Japan Railway Journal',
|
||||
'description': 'md5:ea39d93af7d05835baadf10d1aae0e3f',
|
||||
'series': 'Japan Railway Journal',
|
||||
},
|
||||
'playlist_mincount': 12,
|
||||
}, {
|
||||
@@ -390,6 +303,7 @@ class NhkVodProgramIE(NhkBaseIE):
|
||||
'id': 'japanrailway',
|
||||
'title': 'Japan Railway Journal',
|
||||
'description': 'md5:ea39d93af7d05835baadf10d1aae0e3f',
|
||||
'series': 'Japan Railway Journal',
|
||||
},
|
||||
'playlist_mincount': 12,
|
||||
}, {
|
||||
@@ -399,17 +313,9 @@ class NhkVodProgramIE(NhkBaseIE):
|
||||
'id': 'livinginjapan',
|
||||
'title': 'Living in Japan',
|
||||
'description': 'md5:665bb36ec2a12c5a7f598ee713fc2b54',
|
||||
'series': 'Living in Japan',
|
||||
},
|
||||
'playlist_mincount': 12,
|
||||
}, {
|
||||
# /tv/ program url
|
||||
'url': 'https://www3.nhk.or.jp/nhkworld/en/tv/designtalksplus/',
|
||||
'info_dict': {
|
||||
'id': 'designtalksplus',
|
||||
'title': 'DESIGN TALKS plus',
|
||||
'description': 'md5:47b3b3a9f10d4ac7b33b53b70a7d2837',
|
||||
},
|
||||
'playlist_mincount': 20,
|
||||
'playlist_mincount': 11,
|
||||
}, {
|
||||
'url': 'https://www3.nhk.or.jp/nhkworld/en/shows/10yearshayaomiyazaki/',
|
||||
'only_matching': True,
|
||||
@@ -430,9 +336,8 @@ class NhkVodProgramIE(NhkBaseIE):
|
||||
program_id, lang, m_type != 'audio', False, episode_type == 'clip')
|
||||
|
||||
def entries():
|
||||
for episode in episodes:
|
||||
if episode_path := episode.get('url'):
|
||||
yield self._extract_episode_info(urljoin(url, episode_path), episode)
|
||||
for episode in traverse_obj(episodes, ('items', lambda _, v: v['url'])):
|
||||
yield self._extract_episode_info(urljoin(url, episode['url']), episode)
|
||||
|
||||
html = self._download_webpage(url, program_id)
|
||||
program_title = self._extract_meta_from_class_elements([
|
||||
@@ -446,7 +351,7 @@ class NhkVodProgramIE(NhkBaseIE):
|
||||
'tAudioProgramMain__info', # /shows/audio/programs/
|
||||
'p-program-description'], html) # /tv/
|
||||
|
||||
return self.playlist_result(entries(), program_id, program_title, program_description)
|
||||
return self.playlist_result(entries(), program_id, program_title, program_description, series=program_title)
|
||||
|
||||
|
||||
class NhkForSchoolBangumiIE(InfoExtractor):
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
from .brightcove import BrightcoveNewIE
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
class NowCanalIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?nowcanal\.pt(?:/[\w-]+)+/detalhe/(?P<id>[\w-]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.nowcanal.pt/ultimas/detalhe/pedro-sousa-hjulmand-pode-ter-uma-saida-limpa-do-sporting-daqui-a-um-ano',
|
||||
'md5': '047f17cb783e66e467d703e704bbc95d',
|
||||
'info_dict': {
|
||||
'id': '6376598467112',
|
||||
'ext': 'mp4',
|
||||
'title': 'Pedro Sousa «Hjulmand pode ter uma saída limpa do Sporting daqui a um ano»',
|
||||
'description': '',
|
||||
'uploader_id': '6108484330001',
|
||||
'duration': 65.237,
|
||||
'thumbnail': r're:^https://.+\.jpg',
|
||||
'timestamp': 1754440620,
|
||||
'upload_date': '20250806',
|
||||
'tags': ['now'],
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.nowcanal.pt/programas/frente-a-frente/detalhe/frente-a-frente-eva-cruzeiro-ps-e-rita-matias-chega',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
_BC_URL_TMPL = 'https://players.brightcove.net/6108484330001/chhIqzukMq_default/index.html?videoId={}'
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
|
||||
video_id = self._search_json(
|
||||
r'videoHandler\.addBrightcoveVideoWithJson\(\[',
|
||||
webpage, 'video data', display_id)['brightcoveVideoId']
|
||||
|
||||
return self.url_result(self._BC_URL_TMPL.format(video_id), BrightcoveNewIE)
|
||||
@@ -1,17 +1,40 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
int_or_none,
|
||||
strip_or_none,
|
||||
parse_iso8601,
|
||||
unescapeHTML,
|
||||
url_or_none,
|
||||
xpath_text,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class NTVRuIE(InfoExtractor):
|
||||
IE_NAME = 'ntv.ru'
|
||||
_VALID_URL = r'https?://(?:www\.)?ntv\.ru/(?:[^/]+/)*(?P<id>[^/?#&]+)'
|
||||
_VALID_URL = r'https?://(?:www\.)?ntv\.ru/(?:[^/#?]+/)*(?P<id>[^/?#&]+)'
|
||||
|
||||
_TESTS = [{
|
||||
# JSON Api is geo restricted
|
||||
'url': 'https://www.ntv.ru/peredacha/svoya_igra/m58980/o818800',
|
||||
'md5': '818962a1b52747d446db7cd5be43e142',
|
||||
'info_dict': {
|
||||
'id': '2520563',
|
||||
'ext': 'mp4',
|
||||
'title': 'Участники: Ирина Петрова, Сергей Коновалов, Кристина Кораблина',
|
||||
'description': 'md5:fcbd21cd45238a940b95550f9e178e3e',
|
||||
'thumbnail': r're:^http://.*\.jpg',
|
||||
'duration': 2462,
|
||||
'view_count': int,
|
||||
'comment_count': int,
|
||||
'tags': ['игры и игрушки'],
|
||||
'timestamp': 1761821096,
|
||||
'upload_date': '20251030',
|
||||
'release_timestamp': 1761821096,
|
||||
'release_date': '20251030',
|
||||
'modified_timestamp': 1761821096,
|
||||
'modified_date': '20251030',
|
||||
},
|
||||
}, {
|
||||
'url': 'http://www.ntv.ru/novosti/863142/',
|
||||
'md5': 'ba7ea172a91cb83eb734cad18c10e723',
|
||||
'info_dict': {
|
||||
@@ -22,31 +45,35 @@ class NTVRuIE(InfoExtractor):
|
||||
'thumbnail': r're:^http://.*\.jpg',
|
||||
'duration': 136,
|
||||
'view_count': int,
|
||||
'comment_count': int,
|
||||
'tags': ['ВМС', 'захват', 'митинги', 'Севастополь', 'Украина'],
|
||||
'timestamp': 1395222013,
|
||||
'upload_date': '20140319',
|
||||
'release_timestamp': 1395222013,
|
||||
'release_date': '20140319',
|
||||
'modified_timestamp': 1395222013,
|
||||
'modified_date': '20140319',
|
||||
},
|
||||
}, {
|
||||
'url': 'http://www.ntv.ru/video/novosti/750370/',
|
||||
'md5': 'adecff79691b4d71e25220a191477124',
|
||||
'info_dict': {
|
||||
'id': '750370',
|
||||
'ext': 'mp4',
|
||||
'title': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
|
||||
'description': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
|
||||
'thumbnail': r're:^http://.*\.jpg',
|
||||
'duration': 172,
|
||||
'view_count': int,
|
||||
},
|
||||
'skip': '404 Not Found',
|
||||
}, {
|
||||
# Requires unescapeHTML
|
||||
'url': 'http://www.ntv.ru/peredacha/segodnya/m23700/o232416',
|
||||
'md5': '82dbd49b38e3af1d00df16acbeab260c',
|
||||
'info_dict': {
|
||||
'id': '747480',
|
||||
'ext': 'mp4',
|
||||
'title': '«Сегодня». 21 марта 2014 года. 16:00',
|
||||
'description': '«Сегодня». 21 марта 2014 года. 16:00',
|
||||
'title': '"Сегодня". 21 марта 2014 года. 16:00 ',
|
||||
'description': 'md5:bed80745ca72af557433195f51a02785',
|
||||
'thumbnail': r're:^http://.*\.jpg',
|
||||
'duration': 1496,
|
||||
'view_count': int,
|
||||
'comment_count': int,
|
||||
'tags': ['Брюссель', 'гражданство', 'ЕС', 'Крым', 'ОСАГО', 'саммит', 'санкции', 'события', 'чиновники', 'рейтинг'],
|
||||
'timestamp': 1395406951,
|
||||
'upload_date': '20140321',
|
||||
'release_timestamp': 1395406951,
|
||||
'release_date': '20140321',
|
||||
'modified_timestamp': 1395406951,
|
||||
'modified_date': '20140321',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.ntv.ru/kino/Koma_film/m70281/o336036/video/',
|
||||
@@ -54,11 +81,19 @@ class NTVRuIE(InfoExtractor):
|
||||
'info_dict': {
|
||||
'id': '1126480',
|
||||
'ext': 'mp4',
|
||||
'title': 'Остросюжетный фильм «Кома»',
|
||||
'description': 'Остросюжетный фильм «Кома»',
|
||||
'title': 'Остросюжетный фильм "Кома"',
|
||||
'description': 'md5:e79ffd0887425a0f05a58885c408d7d8',
|
||||
'thumbnail': r're:^http://.*\.jpg',
|
||||
'duration': 5592,
|
||||
'duration': 5608,
|
||||
'view_count': int,
|
||||
'comment_count': int,
|
||||
'tags': ['кино'],
|
||||
'timestamp': 1432868572,
|
||||
'upload_date': '20150529',
|
||||
'release_timestamp': 1432868572,
|
||||
'release_date': '20150529',
|
||||
'modified_timestamp': 1432868572,
|
||||
'modified_date': '20150529',
|
||||
},
|
||||
}, {
|
||||
'url': 'http://www.ntv.ru/serial/Delo_vrachey/m31760/o233916/',
|
||||
@@ -66,11 +101,19 @@ class NTVRuIE(InfoExtractor):
|
||||
'info_dict': {
|
||||
'id': '751482',
|
||||
'ext': 'mp4',
|
||||
'title': '«Дело врачей»: «Деревце жизни»',
|
||||
'description': '«Дело врачей»: «Деревце жизни»',
|
||||
'title': '"Дело врачей": "Деревце жизни"',
|
||||
'description': 'md5:d6fbf9193f880f50d9cbfbcc954161c1',
|
||||
'thumbnail': r're:^http://.*\.jpg',
|
||||
'duration': 2590,
|
||||
'view_count': int,
|
||||
'comment_count': int,
|
||||
'tags': ['врачи', 'больницы'],
|
||||
'timestamp': 1395882300,
|
||||
'upload_date': '20140327',
|
||||
'release_timestamp': 1395882300,
|
||||
'release_date': '20140327',
|
||||
'modified_timestamp': 1395882300,
|
||||
'modified_date': '20140327',
|
||||
},
|
||||
}, {
|
||||
# Schemeless file URL
|
||||
@@ -78,48 +121,26 @@ class NTVRuIE(InfoExtractor):
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
_VIDEO_ID_REGEXES = [
|
||||
r'<meta property="og:url" content="https?://www\.ntv\.ru/video/(\d+)',
|
||||
r'<meta property="og:video:(?:url|iframe)" content="https?://www\.ntv\.ru/embed/(\d+)',
|
||||
r'<video embed=[^>]+><id>(\d+)</id>',
|
||||
r'<video restriction[^>]+><key>(\d+)</key>',
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
video_url = self._og_search_property(
|
||||
('video', 'video:iframe'), webpage, default=None)
|
||||
if video_url:
|
||||
video_id = self._search_regex(
|
||||
r'https?://(?:www\.)?ntv\.ru/video/(?:embed/)?(\d+)',
|
||||
video_url, 'video id', default=None)
|
||||
|
||||
if not video_id:
|
||||
video_id = self._html_search_regex(
|
||||
self._VIDEO_ID_REGEXES, webpage, 'video id')
|
||||
video_id = self._html_search_regex(
|
||||
r'<meta property="ya:ovs:feed_url" content="https?://www\.ntv\.ru/(?:exp/)?video/(\d+)', webpage, 'video id')
|
||||
|
||||
player = self._download_xml(
|
||||
f'http://www.ntv.ru/vi{video_id}/',
|
||||
video_id, 'Downloading video XML')
|
||||
|
||||
title = strip_or_none(unescapeHTML(xpath_text(player, './data/title', 'title', fatal=True)))
|
||||
|
||||
video = player.find('./data/video')
|
||||
|
||||
formats = []
|
||||
for format_id in ['', 'hi', 'webm']:
|
||||
file_ = xpath_text(video, f'./{format_id}file')
|
||||
if not file_:
|
||||
video_url = url_or_none(xpath_text(video, f'./{format_id}file'))
|
||||
if not video_url:
|
||||
continue
|
||||
if file_.startswith('//'):
|
||||
file_ = self._proto_relative_url(file_)
|
||||
elif not file_.startswith('http'):
|
||||
file_ = 'http://media.ntv.ru/vod/' + file_
|
||||
formats.append({
|
||||
'url': file_,
|
||||
'url': video_url,
|
||||
'filesize': int_or_none(xpath_text(video, f'./{format_id}size')),
|
||||
})
|
||||
hls_manifest = xpath_text(video, './playback/hls')
|
||||
@@ -131,12 +152,28 @@ class NTVRuIE(InfoExtractor):
|
||||
formats.extend(self._extract_mpd_formats(
|
||||
dash_manifest, video_id, mpd_id='dash', fatal=False))
|
||||
|
||||
metadata = self._download_xml(
|
||||
f'https://www.ntv.ru/exp/video/{video_id}', video_id, 'Downloading XML metadata', fatal=False)
|
||||
|
||||
return {
|
||||
'id': xpath_text(video, './id'),
|
||||
'title': title,
|
||||
'description': strip_or_none(unescapeHTML(xpath_text(player, './data/description'))),
|
||||
'thumbnail': xpath_text(video, './splash'),
|
||||
'duration': int_or_none(xpath_text(video, './totaltime')),
|
||||
'view_count': int_or_none(xpath_text(video, './views')),
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
**traverse_obj(player, {
|
||||
'title': ('data/title/text()', ..., {str}, {unescapeHTML}, any),
|
||||
'description': ('data/description/text()', ..., {str}, {unescapeHTML}, any),
|
||||
'duration': ('data/video/totaltime/text()', ..., {int_or_none}, any),
|
||||
'view_count': ('data/video/views/text()', ..., {int_or_none}, any),
|
||||
'thumbnail': ('data/video/splash/text()', ..., {url_or_none}, any),
|
||||
}),
|
||||
**traverse_obj(metadata, {
|
||||
'title': ('{*}title/text()', ..., {str}, {unescapeHTML}, any),
|
||||
'description': ('{*}description/text()', ..., {str}, {unescapeHTML}, any),
|
||||
'duration': ('{*}duration/text()', ..., {int_or_none}, any),
|
||||
'timestamp': ('{*}create_date/text()', ..., {parse_iso8601}, any),
|
||||
'release_timestamp': ('{*}upload_date/text()', ..., {parse_iso8601}, any),
|
||||
'modified_timestamp': ('{*}modify_date/text()', ..., {parse_iso8601}, any),
|
||||
'tags': ('{*}tag/text()', ..., {str}, {lambda x: x.split(',')}, ..., {str.strip}, filter),
|
||||
'view_count': ('{*}stats/views_total/text()', ..., {int_or_none}, any),
|
||||
'comment_count': ('{*}stats/comments/text()', ..., {int_or_none}, any),
|
||||
}),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
UserNotLive,
|
||||
filter_dict,
|
||||
int_or_none,
|
||||
join_nonempty,
|
||||
parse_iso8601,
|
||||
url_or_none,
|
||||
urlencode_postdata,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class PandaTvIE(InfoExtractor):
|
||||
IE_DESC = 'pandalive.co.kr (팬더티비)'
|
||||
_VALID_URL = r'https?://(?:www\.|m\.)?pandalive\.co\.kr/play/(?P<id>\w+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.pandalive.co.kr/play/bebenim',
|
||||
'info_dict': {
|
||||
'id': 'bebenim',
|
||||
'ext': 'mp4',
|
||||
'channel': '릴리ෆ',
|
||||
'title': r're:앙앙❤ \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
|
||||
'thumbnail': r're:https://cdn\.pandalive\.co\.kr/ivs/v1/.+/thumb\.jpg',
|
||||
'concurrent_view_count': int,
|
||||
'like_count': int,
|
||||
'live_status': 'is_live',
|
||||
'upload_date': str,
|
||||
},
|
||||
'skip': 'The channel is not currently live',
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
channel_id = self._match_id(url)
|
||||
video_meta = self._download_json(
|
||||
'https://api.pandalive.co.kr/v1/live/play', channel_id,
|
||||
'Downloading video meta data', 'Unable to download video meta data',
|
||||
data=urlencode_postdata(filter_dict({
|
||||
'action': 'watch',
|
||||
'userId': channel_id,
|
||||
'password': self.get_param('videopassword'),
|
||||
})), expected_status=400)
|
||||
|
||||
if error_code := traverse_obj(video_meta, ('errorData', 'code', {str})):
|
||||
if error_code == 'castEnd':
|
||||
raise UserNotLive(video_id=channel_id)
|
||||
elif error_code == 'needAdult':
|
||||
self.raise_login_required('Adult verification is required for this stream')
|
||||
elif error_code == 'needLogin':
|
||||
self.raise_login_required('Login is required for this stream')
|
||||
elif error_code == 'needCoinPurchase':
|
||||
raise ExtractorError('Coin purchase is required for this stream', expected=True)
|
||||
elif error_code == 'needUnlimitItem':
|
||||
raise ExtractorError('Ticket purchase is required for this stream', expected=True)
|
||||
elif error_code == 'needPw':
|
||||
raise ExtractorError('Password protected video, use --video-password <password>', expected=True)
|
||||
elif error_code == 'wrongPw':
|
||||
raise ExtractorError('Wrong password', expected=True)
|
||||
else:
|
||||
error_msg = video_meta.get('message')
|
||||
raise ExtractorError(join_nonempty(
|
||||
'API returned error code', error_code,
|
||||
error_msg and 'with error message:', error_msg,
|
||||
delim=' '))
|
||||
|
||||
http_headers = {'Origin': 'https://www.pandalive.co.kr'}
|
||||
|
||||
return {
|
||||
'id': channel_id,
|
||||
'is_live': True,
|
||||
'formats': self._extract_m3u8_formats(
|
||||
video_meta['PlayList']['hls'][0]['url'], channel_id, 'mp4', headers=http_headers, live=True),
|
||||
'http_headers': http_headers,
|
||||
**traverse_obj(video_meta, ('media', {
|
||||
'title': ('title', {str}),
|
||||
'release_timestamp': ('startTime', {parse_iso8601(delim=' ')}),
|
||||
'thumbnail': ('ivsThumbnail', {url_or_none}),
|
||||
'channel': ('userNick', {str}),
|
||||
'concurrent_view_count': ('user', {int_or_none}),
|
||||
'like_count': ('likeCnt', {int_or_none}),
|
||||
})),
|
||||
}
|
||||
@@ -6,7 +6,10 @@ from ..utils.traversal import traverse_obj
|
||||
class PartiBaseIE(InfoExtractor):
|
||||
def _call_api(self, path, video_id, note=None):
|
||||
return self._download_json(
|
||||
f'https://api-backend.parti.com/parti_v2/profile/{path}', video_id, note)
|
||||
f'https://prod-api.parti.com/parti_v2/profile/{path}', video_id, note, headers={
|
||||
'Origin': 'https://parti.com',
|
||||
'Referer': 'https://parti.com/',
|
||||
})
|
||||
|
||||
|
||||
class PartiVideoIE(PartiBaseIE):
|
||||
@@ -20,7 +23,7 @@ class PartiVideoIE(PartiBaseIE):
|
||||
'title': 'NOW LIVE ',
|
||||
'upload_date': '20250327',
|
||||
'categories': ['Gaming'],
|
||||
'thumbnail': 'https://assets.parti.com/351424_eb9e5250-2821-484a-9c5f-ca99aa666c87.png',
|
||||
'thumbnail': 'https://media.parti.com/351424_eb9e5250-2821-484a-9c5f-ca99aa666c87.png',
|
||||
'channel': 'ItZTMGG',
|
||||
'timestamp': 1743044379,
|
||||
},
|
||||
@@ -34,7 +37,7 @@ class PartiVideoIE(PartiBaseIE):
|
||||
return {
|
||||
'id': video_id,
|
||||
'formats': self._extract_m3u8_formats(
|
||||
urljoin('https://watch.parti.com', data['livestream_recording']), video_id, 'mp4'),
|
||||
urljoin('https://media.parti.com/', data['livestream_recording']), video_id, 'mp4'),
|
||||
**traverse_obj(data, {
|
||||
'title': ('event_title', {str}),
|
||||
'channel': ('user_name', {str}),
|
||||
@@ -47,32 +50,27 @@ class PartiVideoIE(PartiBaseIE):
|
||||
|
||||
class PartiLivestreamIE(PartiBaseIE):
|
||||
IE_NAME = 'parti:livestream'
|
||||
_VALID_URL = r'https?://(?:www\.)?parti\.com/creator/(?P<service>[\w]+)/(?P<id>[\w/-]+)'
|
||||
_VALID_URL = r'https?://(?:www\.)?parti\.com/(?!video/)(?P<id>[\w/-]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://parti.com/creator/parti/Capt_Robs_Adventures',
|
||||
'url': 'https://parti.com/247CryptoTracker',
|
||||
'info_dict': {
|
||||
'id': 'Capt_Robs_Adventures',
|
||||
'ext': 'mp4',
|
||||
'id': '247CryptoTracker',
|
||||
'description': 'md5:a78051f3d7e66e6a64c6b1eaf59fd364',
|
||||
'title': r"re:I'm Live on Parti \d{4}-\d{2}-\d{2} \d{2}:\d{2}",
|
||||
'view_count': int,
|
||||
'thumbnail': r're:https://assets\.parti\.com/.+\.png',
|
||||
'timestamp': 1743879776,
|
||||
'upload_date': '20250405',
|
||||
'thumbnail': r're:https://media\.parti\.com/stream-screenshots/.+\.png',
|
||||
'live_status': 'is_live',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}, {
|
||||
'url': 'https://parti.com/creator/discord/sazboxgaming/0',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
service, creator_slug = self._match_valid_url(url).group('service', 'id')
|
||||
creator_slug = self._match_id(url)
|
||||
|
||||
encoded_creator_slug = creator_slug.replace('/', '%23')
|
||||
creator_id = self._call_api(
|
||||
f'get_user_by_social_media/{service}/{encoded_creator_slug}',
|
||||
creator_slug, note='Fetching user ID')
|
||||
f'user_id_from_name/{encoded_creator_slug}',
|
||||
creator_slug, note='Fetching user ID')['user_id']
|
||||
|
||||
data = self._call_api(
|
||||
f'get_livestream_channel_info/{creator_id}', creator_id,
|
||||
@@ -85,11 +83,7 @@ class PartiLivestreamIE(PartiBaseIE):
|
||||
|
||||
return {
|
||||
'id': creator_slug,
|
||||
'formats': self._extract_m3u8_formats(
|
||||
channel_info['playback_url'], creator_slug, live=True, query={
|
||||
'token': channel_info['playback_auth_token'],
|
||||
'player_version': '1.17.0',
|
||||
}),
|
||||
'formats': self._extract_m3u8_formats(channel_info['playback_url'], creator_slug, live=True),
|
||||
'is_live': True,
|
||||
**traverse_obj(data, {
|
||||
'title': ('livestream_event_info', 'event_name', {str}),
|
||||
|
||||
@@ -598,7 +598,8 @@ class PatreonCampaignIE(PatreonBaseIE):
|
||||
'props', 'pageProps', 'bootstrapEnvelope', 'pageBootstrap', 'campaign', 'data', 'id', {str}))
|
||||
if not campaign_id:
|
||||
campaign_id = traverse_obj(self._search_nextjs_v13_data(webpage, vanity), (
|
||||
lambda _, v: v['type'] == 'campaign', 'id', {str}, any, {require('campaign ID')}))
|
||||
((..., 'value', 'campaign', 'data'), lambda _, v: v['type'] == 'campaign'),
|
||||
'id', {str}, any, {require('campaign ID')}))
|
||||
|
||||
params = {
|
||||
'json-api-use-default-includes': 'false',
|
||||
|
||||
@@ -4,6 +4,7 @@ from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
str_or_none,
|
||||
strip_or_none,
|
||||
traverse_obj,
|
||||
update_url,
|
||||
)
|
||||
@@ -50,7 +51,6 @@ class PicartoIE(InfoExtractor):
|
||||
|
||||
if metadata.get('online') == 0:
|
||||
raise ExtractorError('Stream is offline', expected=True)
|
||||
title = metadata['title']
|
||||
|
||||
cdn_data = self._download_json(''.join((
|
||||
update_url(data['getLoadBalancerUrl']['url'], scheme='https'),
|
||||
@@ -79,7 +79,7 @@ class PicartoIE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': channel_id,
|
||||
'title': title.strip(),
|
||||
'title': strip_or_none(metadata.get('title')),
|
||||
'is_live': True,
|
||||
'channel': channel_id,
|
||||
'channel_id': metadata.get('id'),
|
||||
@@ -159,7 +159,7 @@ class PicartoVodIE(InfoExtractor):
|
||||
'id': video_id,
|
||||
**traverse_obj(data, {
|
||||
'id': ('id', {str_or_none}),
|
||||
'title': ('title', {str}),
|
||||
'title': ('title', {str.strip}),
|
||||
'thumbnail': 'video_recording_image_url',
|
||||
'channel': ('channel', 'name', {str}),
|
||||
'age_limit': ('adult', {lambda x: 18 if x else 0}),
|
||||
|
||||
@@ -24,6 +24,7 @@ from ..utils import (
|
||||
url_or_none,
|
||||
urlencode_postdata,
|
||||
)
|
||||
from ..utils.traversal import find_elements, traverse_obj
|
||||
|
||||
|
||||
class PornHubBaseIE(InfoExtractor):
|
||||
@@ -137,23 +138,24 @@ class PornHubIE(PornHubBaseIE):
|
||||
_EMBED_REGEX = [r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?pornhub(?:premium)?\.(?:com|net|org)/embed/[\da-z]+)']
|
||||
_TESTS = [{
|
||||
'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
|
||||
'md5': 'a6391306d050e4547f62b3f485dd9ba9',
|
||||
'md5': '4d4a4e9178b655776f86cf89ecaf0edf',
|
||||
'info_dict': {
|
||||
'id': '648719015',
|
||||
'ext': 'mp4',
|
||||
'title': 'Seductive Indian beauty strips down and fingers her pink pussy',
|
||||
'uploader': 'Babes',
|
||||
'uploader': 'BABES-COM',
|
||||
'uploader_id': '/users/babes-com',
|
||||
'upload_date': '20130628',
|
||||
'timestamp': 1372447216,
|
||||
'duration': 361,
|
||||
'view_count': int,
|
||||
'like_count': int,
|
||||
'dislike_count': int,
|
||||
'comment_count': int,
|
||||
'age_limit': 18,
|
||||
'tags': list,
|
||||
'categories': list,
|
||||
'cast': list,
|
||||
'thumbnail': r're:https?://.+',
|
||||
},
|
||||
}, {
|
||||
# non-ASCII title
|
||||
@@ -480,13 +482,6 @@ class PornHubIE(PornHubBaseIE):
|
||||
comment_count = self._extract_count(
|
||||
r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
|
||||
|
||||
def extract_list(meta_key):
|
||||
div = self._search_regex(
|
||||
rf'(?s)<div[^>]+\bclass=["\'].*?\b{meta_key}Wrapper[^>]*>(.+?)</div>',
|
||||
webpage, meta_key, default=None)
|
||||
if div:
|
||||
return [clean_html(x).strip() for x in re.findall(r'(?s)<a[^>]+\bhref=[^>]+>.+?</a>', div)]
|
||||
|
||||
info = self._search_json_ld(webpage, video_id, default={})
|
||||
# description provided in JSON-LD is irrelevant
|
||||
info['description'] = None
|
||||
@@ -505,9 +500,11 @@ class PornHubIE(PornHubBaseIE):
|
||||
'comment_count': comment_count,
|
||||
'formats': formats,
|
||||
'age_limit': 18,
|
||||
'tags': extract_list('tags'),
|
||||
'categories': extract_list('categories'),
|
||||
'cast': extract_list('pornstars'),
|
||||
**traverse_obj(webpage, {
|
||||
'tags': ({find_elements(attr='data-label', value='tag')}, ..., {clean_html}),
|
||||
'categories': ({find_elements(attr='data-label', value='category')}, ..., {clean_html}),
|
||||
'cast': ({find_elements(attr='data-label', value='pornstar')}, ..., {clean_html}),
|
||||
}),
|
||||
'subtitles': subtitles,
|
||||
}, info)
|
||||
|
||||
|
||||
@@ -3,12 +3,14 @@ from ..utils import (
|
||||
MEDIA_EXTENSIONS,
|
||||
determine_ext,
|
||||
parse_iso8601,
|
||||
traverse_obj,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class RinseFMBaseIE(InfoExtractor):
|
||||
_API_BASE = 'https://rinse.fm/api/query/v1'
|
||||
|
||||
@staticmethod
|
||||
def _parse_entry(entry):
|
||||
return {
|
||||
@@ -45,8 +47,10 @@ class RinseFMIE(RinseFMBaseIE):
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
entry = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['entry']
|
||||
|
||||
entry = self._download_json(
|
||||
f'{self._API_BASE}/episodes/{display_id}', display_id,
|
||||
note='Downloading episode data from API')['entry']
|
||||
|
||||
return self._parse_entry(entry)
|
||||
|
||||
@@ -58,32 +62,35 @@ class RinseFMArtistPlaylistIE(RinseFMBaseIE):
|
||||
'info_dict': {
|
||||
'id': 'resources',
|
||||
'title': '[re]sources',
|
||||
'description': '[re]sources est un label parisien piloté par le DJ et producteur Tommy Kid.',
|
||||
'description': 'md5:fd6a7254e8273510e6d49fbf50edf392',
|
||||
},
|
||||
'playlist_mincount': 40,
|
||||
}, {
|
||||
'url': 'https://rinse.fm/shows/ivy/',
|
||||
'url': 'https://www.rinse.fm/shows/esk',
|
||||
'info_dict': {
|
||||
'id': 'ivy',
|
||||
'title': '[IVY]',
|
||||
'description': 'A dedicated space for DNB/Turbo House and 4x4.',
|
||||
'id': 'esk',
|
||||
'title': 'Esk',
|
||||
'description': 'md5:5893d7c1d411ae8dea7fba12f109aa98',
|
||||
},
|
||||
'playlist_mincount': 7,
|
||||
'playlist_mincount': 139,
|
||||
}]
|
||||
|
||||
def _entries(self, data):
|
||||
for episode in traverse_obj(data, (
|
||||
'props', 'pageProps', 'episodes', lambda _, v: determine_ext(v['fileUrl']) in MEDIA_EXTENSIONS.audio),
|
||||
'episodes', lambda _, v: determine_ext(v['fileUrl']) in MEDIA_EXTENSIONS.audio),
|
||||
):
|
||||
yield self._parse_entry(episode)
|
||||
|
||||
def _real_extract(self, url):
|
||||
playlist_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, playlist_id)
|
||||
title = self._og_search_title(webpage) or self._html_search_meta('title', webpage)
|
||||
description = self._og_search_description(webpage) or self._html_search_meta(
|
||||
'description', webpage)
|
||||
data = self._search_nextjs_data(webpage, playlist_id)
|
||||
|
||||
api_data = self._download_json(
|
||||
f'{self._API_BASE}/shows/{playlist_id}', playlist_id,
|
||||
note='Downloading show data from API')
|
||||
|
||||
return self.playlist_result(
|
||||
self._entries(data), playlist_id, title, description=description)
|
||||
self._entries(api_data), playlist_id,
|
||||
**traverse_obj(api_data, ('entry', {
|
||||
'title': ('title', {str}),
|
||||
'description': ('description', {str}),
|
||||
})))
|
||||
|
||||
@@ -15,14 +15,15 @@ class S4CIE(InfoExtractor):
|
||||
'thumbnail': 'https://www.s4c.cymru/amg/1920x1080/Y_Swn_2023S4C_099_ii.jpg',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.s4c.cymru/clic/programme/856636948',
|
||||
# Geo restricted to the UK
|
||||
'url': 'https://www.s4c.cymru/clic/programme/886303048',
|
||||
'info_dict': {
|
||||
'id': '856636948',
|
||||
'id': '886303048',
|
||||
'ext': 'mp4',
|
||||
'title': 'Am Dro',
|
||||
'title': 'Pennod 1',
|
||||
'description': 'md5:7e3f364b70f61fcdaa8b4cb4a3eb3e7a',
|
||||
'duration': 2880,
|
||||
'description': 'md5:100d8686fc9a632a0cb2db52a3433ffe',
|
||||
'thumbnail': 'https://www.s4c.cymru/amg/1920x1080/Am_Dro_2022-23S4C_P6_4005.jpg',
|
||||
'thumbnail': 'https://www.s4c.cymru/amg/1920x1080/Stad_2025S4C_P1_210053.jpg',
|
||||
},
|
||||
}]
|
||||
|
||||
@@ -51,7 +52,7 @@ class S4CIE(InfoExtractor):
|
||||
'https://player-api.s4c-cdn.co.uk/streaming-urls/prod', video_id, query={
|
||||
'mode': 'od',
|
||||
'application': 'clic',
|
||||
'region': 'WW',
|
||||
'region': 'UK' if player_config.get('application') == 's4chttpl' else 'WW',
|
||||
'extra': 'false',
|
||||
'thirdParty': 'false',
|
||||
'filename': player_config['filename'],
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
decode_packed_codes,
|
||||
urlencode_postdata,
|
||||
)
|
||||
|
||||
|
||||
class SCTEBaseIE(InfoExtractor):
|
||||
_LOGIN_URL = 'https://www.scte.org/SCTE/Sign_In.aspx'
|
||||
_NETRC_MACHINE = 'scte'
|
||||
|
||||
def _perform_login(self, username, password):
|
||||
login_popup = self._download_webpage(
|
||||
self._LOGIN_URL, None, 'Downloading login popup')
|
||||
|
||||
def is_logged(webpage):
|
||||
return any(re.search(p, webpage) for p in (
|
||||
r'class=["\']welcome\b', r'>Sign Out<'))
|
||||
|
||||
# already logged in
|
||||
if is_logged(login_popup):
|
||||
return
|
||||
|
||||
login_form = self._hidden_inputs(login_popup)
|
||||
|
||||
login_form.update({
|
||||
'ctl01$TemplateBody$WebPartManager1$gwpciNewContactSignInCommon$ciNewContactSignInCommon$signInUserName': username,
|
||||
'ctl01$TemplateBody$WebPartManager1$gwpciNewContactSignInCommon$ciNewContactSignInCommon$signInPassword': password,
|
||||
'ctl01$TemplateBody$WebPartManager1$gwpciNewContactSignInCommon$ciNewContactSignInCommon$RememberMe': 'on',
|
||||
})
|
||||
|
||||
response = self._download_webpage(
|
||||
self._LOGIN_URL, None, 'Logging in',
|
||||
data=urlencode_postdata(login_form))
|
||||
|
||||
if '|pageRedirect|' not in response and not is_logged(response):
|
||||
error = self._html_search_regex(
|
||||
r'(?s)<[^>]+class=["\']AsiError["\'][^>]*>(.+?)</',
|
||||
response, 'error message', default=None)
|
||||
if error:
|
||||
raise ExtractorError(f'Unable to login: {error}', expected=True)
|
||||
raise ExtractorError('Unable to log in')
|
||||
|
||||
|
||||
class SCTEIE(SCTEBaseIE):
|
||||
_WORKING = False
|
||||
_VALID_URL = r'https?://learning\.scte\.org/mod/scorm/view\.php?.*?\bid=(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://learning.scte.org/mod/scorm/view.php?id=31484',
|
||||
'info_dict': {
|
||||
'title': 'Introduction to DOCSIS Engineering Professional',
|
||||
'id': '31484',
|
||||
},
|
||||
'playlist_count': 5,
|
||||
'skip': 'Requires account credentials',
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
title = self._search_regex(r'<h1>(.+?)</h1>', webpage, 'title')
|
||||
|
||||
context_id = self._search_regex(r'context-(\d+)', webpage, video_id)
|
||||
content_base = f'https://learning.scte.org/pluginfile.php/{context_id}/mod_scorm/content/8/'
|
||||
context = decode_packed_codes(self._download_webpage(
|
||||
f'{content_base}mobile/data.js', video_id))
|
||||
|
||||
data = self._parse_xml(
|
||||
self._search_regex(
|
||||
r'CreateData\(\s*"(.+?)"', context, 'data').replace(r"\'", "'"),
|
||||
video_id)
|
||||
|
||||
entries = []
|
||||
for asset in data.findall('.//asset'):
|
||||
asset_url = asset.get('url')
|
||||
if not asset_url or not asset_url.endswith('.mp4'):
|
||||
continue
|
||||
asset_id = self._search_regex(
|
||||
r'video_([^_]+)_', asset_url, 'asset id', default=None)
|
||||
if not asset_id:
|
||||
continue
|
||||
entries.append({
|
||||
'id': asset_id,
|
||||
'title': title,
|
||||
'url': content_base + asset_url,
|
||||
})
|
||||
|
||||
return self.playlist_result(entries, video_id, title)
|
||||
|
||||
|
||||
class SCTECourseIE(SCTEBaseIE):
|
||||
_WORKING = False
|
||||
_VALID_URL = r'https?://learning\.scte\.org/(?:mod/sub)?course/view\.php?.*?\bid=(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://learning.scte.org/mod/subcourse/view.php?id=31491',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://learning.scte.org/course/view.php?id=3639',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://learning.scte.org/course/view.php?id=3073',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
course_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, course_id)
|
||||
|
||||
title = self._search_regex(
|
||||
r'<h1>(.+?)</h1>', webpage, 'title', default=None)
|
||||
|
||||
entries = []
|
||||
for mobj in re.finditer(
|
||||
r'''(?x)
|
||||
<a[^>]+
|
||||
href=(["\'])
|
||||
(?P<url>
|
||||
https?://learning\.scte\.org/mod/
|
||||
(?P<kind>scorm|subcourse)/view\.php?(?:(?!\1).)*?
|
||||
\bid=\d+
|
||||
)
|
||||
''',
|
||||
webpage):
|
||||
item_url = mobj.group('url')
|
||||
if item_url == url:
|
||||
continue
|
||||
ie = (SCTEIE.ie_key() if mobj.group('kind') == 'scorm'
|
||||
else SCTECourseIE.ie_key())
|
||||
entries.append(self.url_result(item_url, ie=ie))
|
||||
|
||||
return self.playlist_result(entries, course_id, title)
|
||||
@@ -1064,7 +1064,7 @@ class SoundcloudRelatedIE(SoundcloudPagedPlaylistBaseIE):
|
||||
|
||||
|
||||
class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
|
||||
_VALID_URL = r'https?://api(?:-v2)?\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
|
||||
_VALID_URL = r'https?://api(?:-v2)?\.soundcloud\.com/playlists/(?:soundcloud(?:%3A|:)playlists(?:%3A|:))?(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
|
||||
IE_NAME = 'soundcloud:playlist'
|
||||
_TESTS = [{
|
||||
'url': 'https://api.soundcloud.com/playlists/4110309',
|
||||
@@ -1079,6 +1079,12 @@ class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
|
||||
'album': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
|
||||
},
|
||||
'playlist_count': 6,
|
||||
}, {
|
||||
'url': 'https://api.soundcloud.com/playlists/soundcloud%3Aplaylists%3A1759227795',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://api.soundcloud.com/playlists/soundcloud:playlists:2104769627?secret_token=s-wmpCLuExeYX',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
|
||||
@@ -8,10 +8,11 @@ from ..utils import (
|
||||
|
||||
|
||||
class SportDeutschlandIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:player\.)?sportdeutschland\.tv/(?P<id>(?:[^/?#]+/)?[^?#/&]+)'
|
||||
IE_NAME = 'sporteurope'
|
||||
_VALID_URL = r'https?://(?:player\.)?sporteurope\.tv/(?P<id>(?:[^/?#]+/)?[^?#/&]+)'
|
||||
_TESTS = [{
|
||||
# Single-part video, direct link
|
||||
'url': 'https://sportdeutschland.tv/rostock-griffins/gfl2-rostock-griffins-vs-elmshorn-fighting-pirates',
|
||||
'url': 'https://sporteurope.tv/rostock-griffins/gfl2-rostock-griffins-vs-elmshorn-fighting-pirates',
|
||||
'md5': '35c11a19395c938cdd076b93bda54cde',
|
||||
'info_dict': {
|
||||
'id': '9f27a97d-1544-4d0b-aa03-48d92d17a03a',
|
||||
@@ -19,9 +20,9 @@ class SportDeutschlandIE(InfoExtractor):
|
||||
'title': 'GFL2: Rostock Griffins vs. Elmshorn Fighting Pirates',
|
||||
'display_id': 'rostock-griffins/gfl2-rostock-griffins-vs-elmshorn-fighting-pirates',
|
||||
'channel': 'Rostock Griffins',
|
||||
'channel_url': 'https://sportdeutschland.tv/rostock-griffins',
|
||||
'channel_url': 'https://sporteurope.tv/rostock-griffins',
|
||||
'live_status': 'was_live',
|
||||
'description': 'md5:60cb00067e55dafa27b0933a43d72862',
|
||||
'description': r're:Video-Livestream des Spiels Rostock Griffins vs\. Elmshorn Fighting Pirates.+',
|
||||
'channel_id': '9635f21c-3f67-4584-9ce4-796e9a47276b',
|
||||
'timestamp': 1749913117,
|
||||
'upload_date': '20250614',
|
||||
@@ -29,16 +30,16 @@ class SportDeutschlandIE(InfoExtractor):
|
||||
},
|
||||
}, {
|
||||
# Single-part video, embedded player link
|
||||
'url': 'https://player.sportdeutschland.tv/9e9619c4-7d77-43c4-926d-49fb57dc06dc',
|
||||
'url': 'https://player.sporteurope.tv/9e9619c4-7d77-43c4-926d-49fb57dc06dc',
|
||||
'info_dict': {
|
||||
'id': '9f27a97d-1544-4d0b-aa03-48d92d17a03a',
|
||||
'ext': 'mp4',
|
||||
'title': 'GFL2: Rostock Griffins vs. Elmshorn Fighting Pirates',
|
||||
'display_id': '9e9619c4-7d77-43c4-926d-49fb57dc06dc',
|
||||
'channel': 'Rostock Griffins',
|
||||
'channel_url': 'https://sportdeutschland.tv/rostock-griffins',
|
||||
'channel_url': 'https://sporteurope.tv/rostock-griffins',
|
||||
'live_status': 'was_live',
|
||||
'description': 'md5:60cb00067e55dafa27b0933a43d72862',
|
||||
'description': r're:Video-Livestream des Spiels Rostock Griffins vs\. Elmshorn Fighting Pirates.+',
|
||||
'channel_id': '9635f21c-3f67-4584-9ce4-796e9a47276b',
|
||||
'timestamp': 1749913117,
|
||||
'upload_date': '20250614',
|
||||
@@ -47,7 +48,7 @@ class SportDeutschlandIE(InfoExtractor):
|
||||
'params': {'skip_download': True},
|
||||
}, {
|
||||
# Multi-part video
|
||||
'url': 'https://sportdeutschland.tv/rhine-ruhr-2025-fisu-world-university-games/volleyball-w-japan-vs-brasilien-halbfinale-2',
|
||||
'url': 'https://sporteurope.tv/rhine-ruhr-2025-fisu-world-university-games/volleyball-w-japan-vs-brasilien-halbfinale-2',
|
||||
'info_dict': {
|
||||
'id': '9f63d737-2444-4e3a-a1ea-840df73fd481',
|
||||
'display_id': 'rhine-ruhr-2025-fisu-world-university-games/volleyball-w-japan-vs-brasilien-halbfinale-2',
|
||||
@@ -55,7 +56,7 @@ class SportDeutschlandIE(InfoExtractor):
|
||||
'description': 'md5:0a17da15e48a687e6019639c3452572b',
|
||||
'channel': 'Rhine-Ruhr 2025 FISU World University Games',
|
||||
'channel_id': '9f5216be-a49d-470b-9a30-4fe9df993334',
|
||||
'channel_url': 'https://sportdeutschland.tv/rhine-ruhr-2025-fisu-world-university-games',
|
||||
'channel_url': 'https://sporteurope.tv/rhine-ruhr-2025-fisu-world-university-games',
|
||||
'live_status': 'was_live',
|
||||
},
|
||||
'playlist_count': 2,
|
||||
@@ -66,7 +67,7 @@ class SportDeutschlandIE(InfoExtractor):
|
||||
'title': 'Volleyball w: Japan vs. Braslien - Halbfinale 2 Part 1',
|
||||
'channel': 'Rhine-Ruhr 2025 FISU World University Games',
|
||||
'channel_id': '9f5216be-a49d-470b-9a30-4fe9df993334',
|
||||
'channel_url': 'https://sportdeutschland.tv/rhine-ruhr-2025-fisu-world-university-games',
|
||||
'channel_url': 'https://sporteurope.tv/rhine-ruhr-2025-fisu-world-university-games',
|
||||
'duration': 14773.0,
|
||||
'timestamp': 1753085197,
|
||||
'upload_date': '20250721',
|
||||
@@ -79,16 +80,17 @@ class SportDeutschlandIE(InfoExtractor):
|
||||
'title': 'Volleyball w: Japan vs. Braslien - Halbfinale 2 Part 2',
|
||||
'channel': 'Rhine-Ruhr 2025 FISU World University Games',
|
||||
'channel_id': '9f5216be-a49d-470b-9a30-4fe9df993334',
|
||||
'channel_url': 'https://sportdeutschland.tv/rhine-ruhr-2025-fisu-world-university-games',
|
||||
'channel_url': 'https://sporteurope.tv/rhine-ruhr-2025-fisu-world-university-games',
|
||||
'duration': 14773.0,
|
||||
'timestamp': 1753128421,
|
||||
'upload_date': '20250721',
|
||||
'live_status': 'was_live',
|
||||
},
|
||||
}],
|
||||
'skip': '404 Not Found',
|
||||
}, {
|
||||
# Livestream
|
||||
'url': 'https://sportdeutschland.tv/dtb/gymnastik-international-tag-1',
|
||||
'url': 'https://sporteurope.tv/dtb/gymnastik-international-tag-1',
|
||||
'info_dict': {
|
||||
'id': '95d71b8a-370a-4b87-ad16-94680da18528',
|
||||
'ext': 'mp4',
|
||||
@@ -96,7 +98,7 @@ class SportDeutschlandIE(InfoExtractor):
|
||||
'display_id': 'dtb/gymnastik-international-tag-1',
|
||||
'channel_id': '936ecef1-2f4a-4e08-be2f-68073cb7ecab',
|
||||
'channel': 'Deutscher Turner-Bund',
|
||||
'channel_url': 'https://sportdeutschland.tv/dtb',
|
||||
'channel_url': 'https://sporteurope.tv/dtb',
|
||||
'description': 'md5:07a885dde5838a6f0796ee21dc3b0c52',
|
||||
'live_status': 'is_live',
|
||||
},
|
||||
@@ -106,9 +108,9 @@ class SportDeutschlandIE(InfoExtractor):
|
||||
def _process_video(self, asset_id, video):
|
||||
is_live = video['type'] == 'mux_live'
|
||||
token = self._download_json(
|
||||
f'https://api.sportdeutschland.tv/api/web/personal/asset-token/{asset_id}',
|
||||
f'https://api.sporteurope.tv/api/web/personal/asset-token/{asset_id}',
|
||||
video['id'], query={'type': video['type'], 'playback_id': video['src']},
|
||||
headers={'Referer': 'https://sportdeutschland.tv/'})['token']
|
||||
headers={'Referer': 'https://sporteurope.tv/'})['token']
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||
f'https://stream.mux.com/{video["src"]}.m3u8?token={token}', video['id'], live=is_live)
|
||||
|
||||
@@ -126,7 +128,7 @@ class SportDeutschlandIE(InfoExtractor):
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
meta = self._download_json(
|
||||
f'https://api.sportdeutschland.tv/api/stateless/frontend/assets/{display_id}',
|
||||
f'https://api.sporteurope.tv/api/stateless/frontend/assets/{display_id}',
|
||||
display_id, query={'access_token': 'true'})
|
||||
|
||||
info = {
|
||||
@@ -139,7 +141,7 @@ class SportDeutschlandIE(InfoExtractor):
|
||||
'channel_id': ('profile', 'id'),
|
||||
'is_live': 'currently_live',
|
||||
'was_live': 'was_live',
|
||||
'channel_url': ('profile', 'slug', {lambda x: f'https://sportdeutschland.tv/{x}'}),
|
||||
'channel_url': ('profile', 'slug', {lambda x: f'https://sporteurope.tv/{x}'}),
|
||||
}, get_all=False),
|
||||
}
|
||||
|
||||
|
||||
@@ -101,8 +101,8 @@ class SproutVideoIE(InfoExtractor):
|
||||
webpage = self._download_webpage(
|
||||
url, video_id, headers=traverse_obj(smuggled_data, {'Referer': 'referer'}))
|
||||
data = self._search_json(
|
||||
r'(?:var|const|let)\s+(?:dat|(?:player|video)Info|)\s*=\s*["\']', webpage, 'player info',
|
||||
video_id, contains_pattern=r'[A-Za-z0-9+/=]+', end_pattern=r'["\'];',
|
||||
r'(?:window\.|(?:var|const|let)\s+)(?:dat|(?:player|video)Info|)\s*=\s*["\']', webpage,
|
||||
'player info', video_id, contains_pattern=r'[A-Za-z0-9+/=]+', end_pattern=r'["\'];',
|
||||
transform_source=lambda x: base64.b64decode(x).decode())
|
||||
|
||||
# SproutVideo may send player info for 'SMPTE Color Monitor Test' [a791d7b71b12ecc52e]
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
import base64
|
||||
import binascii
|
||||
import functools
|
||||
import re
|
||||
import urllib.parse
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..dependencies import Cryptodome
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
OnDemandPagedList,
|
||||
clean_html,
|
||||
extract_attributes,
|
||||
urljoin,
|
||||
)
|
||||
from ..utils.traversal import (
|
||||
find_element,
|
||||
find_elements,
|
||||
require,
|
||||
traverse_obj,
|
||||
)
|
||||
|
||||
|
||||
class TarangPlusBaseIE(InfoExtractor):
|
||||
_BASE_URL = 'https://tarangplus.in'
|
||||
|
||||
|
||||
class TarangPlusVideoIE(TarangPlusBaseIE):
|
||||
IE_NAME = 'tarangplus:video'
|
||||
_VALID_URL = r'https?://(?:www\.)?tarangplus\.in/(?:movies|[^#?/]+/[^#?/]+)/(?!episodes)(?P<id>[^#?/]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://tarangplus.in/tarangaplus-originals/khitpit/khitpit-ep-10',
|
||||
'md5': '78ce056cee755687b8a48199909ecf53',
|
||||
'info_dict': {
|
||||
'id': '67b8206719521d054c0059b7',
|
||||
'display_id': 'khitpit-ep-10',
|
||||
'ext': 'mp4',
|
||||
'title': 'Khitpit Ep-10',
|
||||
'description': 'md5:a45b805cb628e15c853d78b0406eab48',
|
||||
'thumbnail': r're:https?://.+/.+\.jpg',
|
||||
'duration': 756.0,
|
||||
'timestamp': 1740355200,
|
||||
'upload_date': '20250224',
|
||||
'media_type': 'episode',
|
||||
'categories': ['Originals'],
|
||||
},
|
||||
}, {
|
||||
'url': 'https://tarangplus.in/tarang-serials/bada-bohu/bada-bohu-ep-233',
|
||||
'md5': 'b4f9beb15172559bb362203b4f48382e',
|
||||
'info_dict': {
|
||||
'id': '680b9d6c19521d054c007782',
|
||||
'display_id': 'bada-bohu-ep-233',
|
||||
'ext': 'mp4',
|
||||
'title': 'Bada Bohu | Ep -233',
|
||||
'description': 'md5:e6b8e7edc9e60b92c1b390f8789ecd69',
|
||||
'thumbnail': r're:https?://.+/.+\.jpg',
|
||||
'duration': 1392.0,
|
||||
'timestamp': 1745539200,
|
||||
'upload_date': '20250425',
|
||||
'media_type': 'episode',
|
||||
'categories': ['Prime'],
|
||||
},
|
||||
}, {
|
||||
# Decrypted m3u8 URL has trailing control characters that need to be stripped
|
||||
'url': 'https://tarangplus.in/tarangaplus-originals/ichha/ichha-teaser-1',
|
||||
'md5': '16ee43fe21ad8b6e652ec65eba38a64e',
|
||||
'info_dict': {
|
||||
'id': '5f0f252d3326af0720000342',
|
||||
'ext': 'mp4',
|
||||
'display_id': 'ichha-teaser-1',
|
||||
'title': 'Ichha Teaser',
|
||||
'description': 'md5:c724b0b0669a2cefdada3711cec792e6',
|
||||
'media_type': 'episode',
|
||||
'duration': 21.0,
|
||||
'thumbnail': r're:https?://.+/.+\.jpg',
|
||||
'categories': ['Originals'],
|
||||
'timestamp': 1758153600,
|
||||
'upload_date': '20250918',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://tarangplus.in/short/ai-maa/ai-maa',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://tarangplus.in/shows/tarang-cine-utsav-2024/tarang-cine-utsav-2024-seg-1',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://tarangplus.in/music-videos/chori-chori-bohu-chori-songs/nijara-laguchu-dhire-dhire',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://tarangplus.in/kids-shows/chhota-jaga/chhota-jaga-ep-33-jamidar-ra-khajana-adaya',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://tarangplus.in/movies/swayambara',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def decrypt(self, data, key):
|
||||
if not Cryptodome.AES:
|
||||
raise ExtractorError('pycryptodomex not found. Please install', expected=True)
|
||||
iv = binascii.unhexlify('00000000000000000000000000000000')
|
||||
cipher = Cryptodome.AES.new(base64.b64decode(key), Cryptodome.AES.MODE_CBC, iv)
|
||||
return cipher.decrypt(base64.b64decode(data)).decode('utf-8')
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
hidden_inputs_data = self._hidden_inputs(webpage)
|
||||
json_ld_data = self._search_json_ld(webpage, display_id)
|
||||
json_ld_data.pop('url', None)
|
||||
|
||||
iframe_url = traverse_obj(webpage, (
|
||||
{find_element(tag='iframe', attr='src', value=r'.+[?&]contenturl=.+', html=True, regex=True)},
|
||||
{extract_attributes}, 'src', {require('iframe URL')}))
|
||||
# Can't use parse_qs here since it would decode the encrypted base64 `+` chars to spaces
|
||||
content = self._search_regex(r'[?&]contenturl=(.+)', iframe_url, 'content')
|
||||
encrypted_data, _, attrs = content.partition('|')
|
||||
metadata = {
|
||||
m.group('k'): m.group('v')
|
||||
for m in re.finditer(r'(?:^|\|)(?P<k>[a-z_]+)=(?P<v>(?:(?!\|[a-z_]+=).)+)', attrs)
|
||||
}
|
||||
m3u8_url = urllib.parse.unquote(
|
||||
self.decrypt(encrypted_data, metadata['key'])).rstrip('\x0e\x0f')
|
||||
|
||||
return {
|
||||
'id': display_id, # Fallback
|
||||
'display_id': display_id,
|
||||
**json_ld_data,
|
||||
**traverse_obj(metadata, {
|
||||
'id': ('content_id', {str}),
|
||||
'title': ('title', {str}),
|
||||
'thumbnail': ('image', {str}),
|
||||
}),
|
||||
**traverse_obj(hidden_inputs_data, {
|
||||
'id': ('content_id', {str}),
|
||||
'media_type': ('theme_type', {str}),
|
||||
'categories': ('genre', {str}, filter, all, filter),
|
||||
}),
|
||||
'formats': self._extract_m3u8_formats(m3u8_url, display_id),
|
||||
}
|
||||
|
||||
|
||||
class TarangPlusEpisodesIE(TarangPlusBaseIE):
|
||||
IE_NAME = 'tarangplus:episodes'
|
||||
_VALID_URL = r'https?://(?:www\.)?tarangplus\.in/(?P<type>[^#?/]+)/(?P<id>[^#?/]+)/episodes/?(?:$|[?#])'
|
||||
_TESTS = [{
|
||||
'url': 'https://tarangplus.in/tarangaplus-originals/balijatra/episodes',
|
||||
'info_dict': {
|
||||
'id': 'balijatra',
|
||||
'title': 'Balijatra',
|
||||
},
|
||||
'playlist_mincount': 7,
|
||||
}, {
|
||||
'url': 'https://tarangplus.in/tarang-serials/bada-bohu/episodes',
|
||||
'info_dict': {
|
||||
'id': 'bada-bohu',
|
||||
'title': 'Bada Bohu',
|
||||
},
|
||||
'playlist_mincount': 236,
|
||||
}, {
|
||||
'url': 'https://tarangplus.in/shows/dr-nonsense/episodes',
|
||||
'info_dict': {
|
||||
'id': 'dr-nonsense',
|
||||
'title': 'Dr. Nonsense',
|
||||
},
|
||||
'playlist_mincount': 15,
|
||||
}]
|
||||
_PAGE_SIZE = 20
|
||||
|
||||
def _entries(self, playlist_url, playlist_id, page):
|
||||
data = self._download_json(
|
||||
playlist_url, playlist_id, f'Downloading playlist JSON page {page + 1}',
|
||||
query={'page_no': page})
|
||||
for item in traverse_obj(data, ('items', ..., {str})):
|
||||
yield self.url_result(
|
||||
urljoin(self._BASE_URL, item.split('$')[3]), TarangPlusVideoIE)
|
||||
|
||||
def _real_extract(self, url):
|
||||
url_type, display_id = self._match_valid_url(url).group('type', 'id')
|
||||
series_url = f'{self._BASE_URL}/{url_type}/{display_id}'
|
||||
webpage = self._download_webpage(series_url, display_id)
|
||||
|
||||
entries = OnDemandPagedList(
|
||||
functools.partial(self._entries, f'{series_url}/episodes', display_id),
|
||||
self._PAGE_SIZE)
|
||||
return self.playlist_result(
|
||||
entries, display_id, self._hidden_inputs(webpage).get('title'))
|
||||
|
||||
|
||||
class TarangPlusPlaylistIE(TarangPlusBaseIE):
|
||||
IE_NAME = 'tarangplus:playlist'
|
||||
_VALID_URL = r'https?://(?:www\.)?tarangplus\.in/(?P<id>[^#?/]+)/all/?(?:$|[?#])'
|
||||
_TESTS = [{
|
||||
'url': 'https://tarangplus.in/chhota-jaga/all',
|
||||
'info_dict': {
|
||||
'id': 'chhota-jaga',
|
||||
'title': 'Chhota Jaga',
|
||||
},
|
||||
'playlist_mincount': 33,
|
||||
}, {
|
||||
'url': 'https://tarangplus.in/kids-yali-show/all',
|
||||
'info_dict': {
|
||||
'id': 'kids-yali-show',
|
||||
'title': 'Yali',
|
||||
},
|
||||
'playlist_mincount': 10,
|
||||
}, {
|
||||
'url': 'https://tarangplus.in/trailer/all',
|
||||
'info_dict': {
|
||||
'id': 'trailer',
|
||||
'title': 'Trailer',
|
||||
},
|
||||
'playlist_mincount': 57,
|
||||
}, {
|
||||
'url': 'https://tarangplus.in/latest-songs/all',
|
||||
'info_dict': {
|
||||
'id': 'latest-songs',
|
||||
'title': 'Latest Songs',
|
||||
},
|
||||
'playlist_mincount': 46,
|
||||
}, {
|
||||
'url': 'https://tarangplus.in/premium-serials-episodes/all',
|
||||
'info_dict': {
|
||||
'id': 'premium-serials-episodes',
|
||||
'title': 'Primetime Latest Episodes',
|
||||
},
|
||||
'playlist_mincount': 100,
|
||||
}]
|
||||
|
||||
def _entries(self, webpage):
|
||||
for url_path in traverse_obj(webpage, (
|
||||
{find_elements(cls='item')}, ...,
|
||||
{find_elements(tag='a', attr='href', value='/.+', html=True, regex=True)},
|
||||
..., {extract_attributes}, 'href',
|
||||
)):
|
||||
yield self.url_result(urljoin(self._BASE_URL, url_path), TarangPlusVideoIE)
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
|
||||
return self.playlist_result(
|
||||
self._entries(webpage), display_id,
|
||||
traverse_obj(webpage, ({find_element(id='al_title')}, {clean_html})))
|
||||
@@ -6,20 +6,21 @@ from ..networking.exceptions import HTTPError
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
clean_html,
|
||||
extract_attributes,
|
||||
int_or_none,
|
||||
join_nonempty,
|
||||
str_or_none,
|
||||
traverse_obj,
|
||||
update_url,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class TelecincoBaseIE(InfoExtractor):
|
||||
def _parse_content(self, content, url):
|
||||
video_id = content['dataMediaId']
|
||||
video_id = content['dataMediaId'][1]
|
||||
config = self._download_json(
|
||||
content['dataConfig'], video_id, 'Downloading config JSON')
|
||||
content['dataConfig'][1], video_id, 'Downloading config JSON')
|
||||
services = config['services']
|
||||
caronte = self._download_json(services['caronte'], video_id)
|
||||
if traverse_obj(caronte, ('dls', 0, 'drm', {bool})):
|
||||
@@ -57,9 +58,9 @@ class TelecincoBaseIE(InfoExtractor):
|
||||
'id': video_id,
|
||||
'title': traverse_obj(config, ('info', 'title', {str})),
|
||||
'formats': formats,
|
||||
'thumbnail': (traverse_obj(content, ('dataPoster', {url_or_none}))
|
||||
'thumbnail': (traverse_obj(content, ('dataPoster', 1, {url_or_none}))
|
||||
or traverse_obj(config, 'poster', 'imageUrl', expected_type=url_or_none)),
|
||||
'duration': traverse_obj(content, ('dataDuration', {int_or_none})),
|
||||
'duration': traverse_obj(content, ('dataDuration', 1, {int_or_none})),
|
||||
'http_headers': headers,
|
||||
}
|
||||
|
||||
@@ -137,30 +138,45 @@ class TelecincoIE(TelecincoBaseIE):
|
||||
'url': 'http://www.cuatro.com/chesterinlove/a-carta/chester-chester_in_love-chester_edu_2_2331030022.html',
|
||||
'only_matching': True,
|
||||
}]
|
||||
_ASTRO_ISLAND_RE = re.compile(r'<astro-island\b[^>]+>')
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id, impersonate=True)
|
||||
article = self._search_json(
|
||||
r'window\.\$REACTBASE_STATE\.article(?:_multisite)?\s*=',
|
||||
webpage, 'article', display_id)['article']
|
||||
description = traverse_obj(article, ('leadParagraph', {clean_html}, filter))
|
||||
|
||||
if article.get('editorialType') != 'VID':
|
||||
props_list = traverse_obj(webpage, (
|
||||
{self._ASTRO_ISLAND_RE.findall}, ...,
|
||||
{extract_attributes}, 'props', {json.loads}))
|
||||
|
||||
description = traverse_obj(props_list, (..., 'leadParagraph', 1, {clean_html}, any, filter))
|
||||
main_content = traverse_obj(props_list, (..., ('content', ('articleData', 1, 'opening')), 1, {dict}, any))
|
||||
|
||||
if traverse_obj(props_list, (..., 'editorialType', 1, {str}, any)) != 'VID': # e.g. 'ART'
|
||||
entries = []
|
||||
|
||||
for p in traverse_obj(article, ((('opening', all), 'body'), lambda _, v: v['content'])):
|
||||
content = p['content']
|
||||
type_ = p.get('type')
|
||||
if type_ == 'paragraph' and isinstance(content, str):
|
||||
for p in traverse_obj(props_list, (..., 'articleData', 1, ('opening', ('body', 1, ...)), 1, {dict})):
|
||||
type_ = traverse_obj(p, ('type', 1, {str}))
|
||||
content = traverse_obj(p, ('content', 1, {str} if type_ == 'paragraph' else {dict}))
|
||||
if not content:
|
||||
continue
|
||||
if type_ == 'paragraph':
|
||||
description = join_nonempty(description, content, delim='')
|
||||
elif type_ == 'video' and isinstance(content, dict):
|
||||
elif type_ == 'video':
|
||||
entries.append(self._parse_content(content, url))
|
||||
else:
|
||||
self.report_warning(
|
||||
f'Skipping unsupported content type "{type_}"', display_id, only_once=True)
|
||||
|
||||
return self.playlist_result(
|
||||
entries, str_or_none(article.get('id')),
|
||||
traverse_obj(article, ('title', {str})), clean_html(description))
|
||||
entries,
|
||||
traverse_obj(props_list, (..., 'id', 1, {int}, {str_or_none}, any)) or display_id,
|
||||
traverse_obj(main_content, ('dataTitle', 1, {str})),
|
||||
clean_html(description))
|
||||
|
||||
info = self._parse_content(article['opening']['content'], url)
|
||||
if not main_content:
|
||||
raise ExtractorError('Unable to extract main content from webpage')
|
||||
|
||||
info = self._parse_content(main_content, url)
|
||||
info['description'] = description
|
||||
|
||||
return info
|
||||
|
||||
@@ -1,18 +1,17 @@
|
||||
import json
|
||||
import urllib.parse
|
||||
|
||||
from .brightcove import BrightcoveNewIE
|
||||
from .common import InfoExtractor
|
||||
from .zype import ZypeIE
|
||||
from ..networking import HEADRequest
|
||||
from ..networking.exceptions import HTTPError
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
filter_dict,
|
||||
parse_qs,
|
||||
smuggle_url,
|
||||
try_call,
|
||||
urlencode_postdata,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class ThisOldHouseIE(InfoExtractor):
|
||||
@@ -77,46 +76,43 @@ class ThisOldHouseIE(InfoExtractor):
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
_LOGIN_URL = 'https://login.thisoldhouse.com/usernamepassword/login'
|
||||
|
||||
def _perform_login(self, username, password):
|
||||
self._request_webpage(
|
||||
HEADRequest('https://www.thisoldhouse.com/insider'), None, 'Requesting session cookies')
|
||||
urlh = self._request_webpage(
|
||||
'https://www.thisoldhouse.com/wp-login.php', None, 'Requesting login info',
|
||||
errnote='Unable to login', query={'redirect_to': 'https://www.thisoldhouse.com/insider'})
|
||||
login_page = self._download_webpage(
|
||||
'https://www.thisoldhouse.com/insider-login', None, 'Downloading login page')
|
||||
hidden_inputs = self._hidden_inputs(login_page)
|
||||
response = self._download_json(
|
||||
'https://www.thisoldhouse.com/wp-admin/admin-ajax.php', None, 'Logging in',
|
||||
headers={
|
||||
'Accept': 'application/json',
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
}, data=urlencode_postdata(filter_dict({
|
||||
'action': 'onebill_subscriber_login',
|
||||
'email': username,
|
||||
'password': password,
|
||||
'pricingPlanTerm': hidden_inputs['pricing_plan_term'],
|
||||
'utm_parameters': hidden_inputs.get('utm_parameters'),
|
||||
'nonce': hidden_inputs['mdcr_onebill_login_nonce'],
|
||||
})))
|
||||
|
||||
try:
|
||||
auth_form = self._download_webpage(
|
||||
self._LOGIN_URL, None, 'Submitting credentials', headers={
|
||||
'Content-Type': 'application/json',
|
||||
'Referer': urlh.url,
|
||||
}, data=json.dumps(filter_dict({
|
||||
**{('client_id' if k == 'client' else k): v[0] for k, v in parse_qs(urlh.url).items()},
|
||||
'tenant': 'thisoldhouse',
|
||||
'username': username,
|
||||
'password': password,
|
||||
'popup_options': {},
|
||||
'sso': True,
|
||||
'_csrf': try_call(lambda: self._get_cookies(self._LOGIN_URL)['_csrf'].value),
|
||||
'_intstate': 'deprecated',
|
||||
}), separators=(',', ':')).encode())
|
||||
except ExtractorError as e:
|
||||
if isinstance(e.cause, HTTPError) and e.cause.status == 401:
|
||||
message = traverse_obj(response, ('data', 'message', {str}))
|
||||
if not response['success']:
|
||||
if message and 'Something went wrong' in message:
|
||||
raise ExtractorError('Invalid username or password', expected=True)
|
||||
raise
|
||||
|
||||
self._request_webpage(
|
||||
'https://login.thisoldhouse.com/login/callback', None, 'Completing login',
|
||||
data=urlencode_postdata(self._hidden_inputs(auth_form)))
|
||||
raise ExtractorError(message or 'Login was unsuccessful')
|
||||
if message and 'Your subscription is not active' in message:
|
||||
self.report_warning(
|
||||
f'{self.IE_NAME} said your subscription is not active. '
|
||||
f'If your subscription is active, this could be caused by too many sign-ins, '
|
||||
f'and you should instead try using {self._login_hint(method="cookies")[4:]}')
|
||||
else:
|
||||
self.write_debug(f'{self.IE_NAME} said: {message}')
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
if 'To Unlock This content' in webpage:
|
||||
self.raise_login_required(
|
||||
'This video is only available for subscribers. '
|
||||
'Note that --cookies-from-browser may not work due to this site using session cookies')
|
||||
webpage, urlh = self._download_webpage_handle(url, display_id)
|
||||
# If login response says inactive subscription, site redirects to frontpage for Insider content
|
||||
if 'To Unlock This content' in webpage or urllib.parse.urlparse(urlh.url).path in ('', '/'):
|
||||
self.raise_login_required('This video is only available for subscribers')
|
||||
|
||||
video_url, video_id = self._search_regex(
|
||||
r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?thisoldhouse\.(?:chorus\.build|com)/videos/zype/([0-9a-f]{24})[^\'"]*)[\'"]',
|
||||
|
||||
@@ -454,6 +454,7 @@ class TikTokBaseIE(InfoExtractor):
|
||||
'like_count': 'digg_count',
|
||||
'repost_count': 'share_count',
|
||||
'comment_count': 'comment_count',
|
||||
'save_count': 'collect_count',
|
||||
}, expected_type=int_or_none),
|
||||
**author_info,
|
||||
'channel_url': format_field(author_info, 'channel_id', self._UPLOADER_URL_FORMAT, default=None),
|
||||
@@ -607,6 +608,7 @@ class TikTokBaseIE(InfoExtractor):
|
||||
'like_count': 'diggCount',
|
||||
'repost_count': 'shareCount',
|
||||
'comment_count': 'commentCount',
|
||||
'save_count': 'collectCount',
|
||||
}), expected_type=int_or_none),
|
||||
'thumbnails': [
|
||||
{
|
||||
@@ -646,6 +648,7 @@ class TikTokIE(TikTokBaseIE):
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
'comment_count': int,
|
||||
'save_count': int,
|
||||
'artist': 'Ysrbeats',
|
||||
'album': 'Lehanga',
|
||||
'track': 'Lehanga',
|
||||
@@ -675,6 +678,7 @@ class TikTokIE(TikTokBaseIE):
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
'comment_count': int,
|
||||
'save_count': int,
|
||||
'artists': ['Evan Todd', 'Jessica Keenan Wynn', 'Alice Lee', 'Barrett Wilbert Weed', 'Jon Eidson'],
|
||||
'track': 'Big Fun',
|
||||
},
|
||||
@@ -702,6 +706,7 @@ class TikTokIE(TikTokBaseIE):
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
'comment_count': int,
|
||||
'save_count': int,
|
||||
},
|
||||
}, {
|
||||
# Sponsored video, only available with feed workaround
|
||||
@@ -725,6 +730,7 @@ class TikTokIE(TikTokBaseIE):
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
'comment_count': int,
|
||||
'save_count': int,
|
||||
},
|
||||
'skip': 'This video is unavailable',
|
||||
}, {
|
||||
@@ -751,6 +757,7 @@ class TikTokIE(TikTokBaseIE):
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
'comment_count': int,
|
||||
'save_count': int,
|
||||
},
|
||||
}, {
|
||||
# hydration JSON is sent in a <script> element
|
||||
@@ -773,6 +780,7 @@ class TikTokIE(TikTokBaseIE):
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
'comment_count': int,
|
||||
'save_count': int,
|
||||
},
|
||||
'skip': 'This video is unavailable',
|
||||
}, {
|
||||
@@ -798,6 +806,7 @@ class TikTokIE(TikTokBaseIE):
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
'comment_count': int,
|
||||
'save_count': int,
|
||||
'thumbnail': r're:^https://.+\.(?:webp|jpe?g)',
|
||||
},
|
||||
}, {
|
||||
@@ -824,6 +833,7 @@ class TikTokIE(TikTokBaseIE):
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
'comment_count': int,
|
||||
'save_count': int,
|
||||
'thumbnail': r're:^https://.+',
|
||||
'thumbnails': 'count:3',
|
||||
},
|
||||
@@ -851,6 +861,7 @@ class TikTokIE(TikTokBaseIE):
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
'comment_count': int,
|
||||
'save_count': int,
|
||||
'thumbnail': r're:^https://.+\.webp',
|
||||
},
|
||||
'skip': 'Unavailable via feed API, only audio available via web',
|
||||
@@ -879,6 +890,7 @@ class TikTokIE(TikTokBaseIE):
|
||||
'like_count': int,
|
||||
'comment_count': int,
|
||||
'repost_count': int,
|
||||
'save_count': int,
|
||||
'thumbnail': r're:^https://.+\.(?:webp|jpe?g)',
|
||||
},
|
||||
}, {
|
||||
@@ -1288,6 +1300,7 @@ class DouyinIE(TikTokBaseIE):
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
'comment_count': int,
|
||||
'save_count': int,
|
||||
'thumbnail': r're:https?://.+\.jpe?g',
|
||||
},
|
||||
}, {
|
||||
@@ -1312,6 +1325,7 @@ class DouyinIE(TikTokBaseIE):
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
'comment_count': int,
|
||||
'save_count': int,
|
||||
'thumbnail': r're:https?://.+\.jpe?g',
|
||||
},
|
||||
}, {
|
||||
@@ -1336,6 +1350,7 @@ class DouyinIE(TikTokBaseIE):
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
'comment_count': int,
|
||||
'save_count': int,
|
||||
'thumbnail': r're:https?://.+\.jpe?g',
|
||||
},
|
||||
}, {
|
||||
@@ -1353,6 +1368,7 @@ class DouyinIE(TikTokBaseIE):
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
'comment_count': int,
|
||||
'save_count': int,
|
||||
},
|
||||
'skip': 'No longer available',
|
||||
}, {
|
||||
@@ -1377,6 +1393,7 @@ class DouyinIE(TikTokBaseIE):
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
'comment_count': int,
|
||||
'save_count': int,
|
||||
'thumbnail': r're:https?://.+\.jpe?g',
|
||||
},
|
||||
}]
|
||||
@@ -1437,6 +1454,7 @@ class TikTokVMIE(InfoExtractor):
|
||||
'view_count': int,
|
||||
'like_count': int,
|
||||
'comment_count': int,
|
||||
'save_count': int,
|
||||
'thumbnail': r're:https://.+\.webp.*',
|
||||
'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAdZ_NcPPgMneaGrW0hN8O_J_bwLshwNNERRF5DxOw2HKIzk0kdlLrR8RkVl1ksrMO',
|
||||
'duration': 29,
|
||||
|
||||
@@ -136,8 +136,10 @@ class TubeTuGrazIE(TubeTuGrazBaseIE):
|
||||
IE_DESC = 'tube.tugraz.at'
|
||||
|
||||
_VALID_URL = r'''(?x)
|
||||
https?://tube\.tugraz\.at/paella/ui/watch.html\?id=
|
||||
(?P<id>[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12})
|
||||
https?://tube\.tugraz\.at/(?:
|
||||
paella/ui/watch\.html\?(?:[^#]*&)?id=|
|
||||
portal/watch/
|
||||
)(?P<id>[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12})
|
||||
'''
|
||||
_TESTS = [
|
||||
{
|
||||
@@ -149,9 +151,9 @@ class TubeTuGrazIE(TubeTuGrazBaseIE):
|
||||
'title': '#6 (23.11.2017)',
|
||||
'episode': '#6 (23.11.2017)',
|
||||
'series': '[INB03001UF] Einführung in die strukturierte Programmierung',
|
||||
'creator': 'Safran C',
|
||||
'duration': 3295818,
|
||||
'series_id': 'b1192fff-2aa7-4bf0-a5cf-7b15c3bd3b34',
|
||||
'creators': ['Safran C'],
|
||||
},
|
||||
}, {
|
||||
'url': 'https://tube.tugraz.at/paella/ui/watch.html?id=2df6d787-e56a-428d-8ef4-d57f07eef238',
|
||||
@@ -162,6 +164,10 @@ class TubeTuGrazIE(TubeTuGrazBaseIE):
|
||||
'ext': 'mp4',
|
||||
},
|
||||
'expected_warnings': ['Extractor failed to obtain "title"'],
|
||||
}, {
|
||||
# Portal URL format
|
||||
'url': 'https://tube.tugraz.at/portal/watch/ab28ec60-8cbe-4f1a-9b96-a95add56c612',
|
||||
'only_matching': True,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from ..utils import (
|
||||
|
||||
class TubiTvIE(InfoExtractor):
|
||||
IE_NAME = 'tubitv'
|
||||
_VALID_URL = r'https?://(?:www\.)?tubitv\.com/(?P<type>video|movies|tv-shows)/(?P<id>\d+)'
|
||||
_VALID_URL = r'https?://(?:www\.)?tubitv\.com/(?:[a-z]{2}-[a-z]{2}/)?(?P<type>video|movies|tv-shows)/(?P<id>\d+)'
|
||||
_LOGIN_URL = 'http://tubitv.com/login'
|
||||
_NETRC_MACHINE = 'tubitv'
|
||||
_TESTS = [{
|
||||
@@ -73,6 +73,9 @@ class TubiTvIE(InfoExtractor):
|
||||
'release_year': 1979,
|
||||
},
|
||||
'skip': 'Content Unavailable',
|
||||
}, {
|
||||
'url': 'https://tubitv.com/es-mx/tv-shows/477363/s01-e03-jacob-dos-dos-y-la-tarjets-de-hockey-robada',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
# DRM formats are included only to raise appropriate error
|
||||
@@ -182,13 +185,13 @@ class TubiTvShowIE(InfoExtractor):
|
||||
webpage = self._download_webpage(show_url, playlist_id)
|
||||
|
||||
data = self._search_json(
|
||||
r'window\.__data\s*=', webpage, 'data', playlist_id,
|
||||
transform_source=js_to_json)['video']
|
||||
r'window\.__REACT_QUERY_STATE__\s*=', webpage, 'data', playlist_id,
|
||||
transform_source=js_to_json)['queries'][0]['state']['data']
|
||||
|
||||
# v['number'] is already a decimal string, but stringify to protect against API changes
|
||||
path = [lambda _, v: str(v['number']) == selected_season] if selected_season else [..., {dict}]
|
||||
|
||||
for season in traverse_obj(data, ('byId', lambda _, v: v['type'] == 's', 'seasons', *path)):
|
||||
for season in traverse_obj(data, ('seasons', *path)):
|
||||
season_number = int_or_none(season.get('number'))
|
||||
for episode in traverse_obj(season, ('episodes', lambda _, v: v['id'])):
|
||||
episode_id = episode['id']
|
||||
|
||||
@@ -20,6 +20,8 @@ class TumblrIE(InfoExtractor):
|
||||
'id': '54196191430',
|
||||
'ext': 'mp4',
|
||||
'title': 'md5:dfac39636969fe6bf1caa2d50405f069',
|
||||
'timestamp': 1372531260,
|
||||
'upload_date': '20130629',
|
||||
'description': 'md5:390ab77358960235b6937ab3b8528956',
|
||||
'uploader_id': 'tatianamaslanydaily',
|
||||
'uploader_url': 'https://tatianamaslanydaily.tumblr.com/',
|
||||
@@ -39,6 +41,8 @@ class TumblrIE(InfoExtractor):
|
||||
'ext': 'mp4',
|
||||
'title': 'Mona\xa0“talking” in\xa0“english”',
|
||||
'description': 'md5:082a3a621530cb786ad2b7592a6d9e2c',
|
||||
'timestamp': 1597865276,
|
||||
'upload_date': '20200819',
|
||||
'uploader_id': 'maskofthedragon',
|
||||
'uploader_url': 'https://maskofthedragon.tumblr.com/',
|
||||
'thumbnail': r're:^https?://.*\.jpg',
|
||||
@@ -76,6 +80,8 @@ class TumblrIE(InfoExtractor):
|
||||
'id': '159704441298',
|
||||
'ext': 'mp4',
|
||||
'title': 'md5:ba79365861101f4911452728d2950561',
|
||||
'timestamp': 1492489550,
|
||||
'upload_date': '20170418',
|
||||
'description': 'md5:773738196cea76b6996ec71e285bdabc',
|
||||
'uploader_id': 'jujanon',
|
||||
'uploader_url': 'https://jujanon.tumblr.com/',
|
||||
@@ -93,6 +99,8 @@ class TumblrIE(InfoExtractor):
|
||||
'id': '180294460076',
|
||||
'ext': 'mp4',
|
||||
'title': 'duality of bird',
|
||||
'timestamp': 1542651819,
|
||||
'upload_date': '20181119',
|
||||
'description': 'duality of bird',
|
||||
'uploader_id': 'todaysbird',
|
||||
'uploader_url': 'https://todaysbird.tumblr.com/',
|
||||
@@ -238,6 +246,8 @@ class TumblrIE(InfoExtractor):
|
||||
'info_dict': {
|
||||
'id': '730460905855467520',
|
||||
'uploader_id': 'felixcosm',
|
||||
'upload_date': '20231006',
|
||||
'timestamp': 1696621805,
|
||||
'repost_count': int,
|
||||
'tags': 'count:15',
|
||||
'description': 'md5:2eb3482a3c6987280cbefb6839068f32',
|
||||
@@ -327,6 +337,8 @@ class TumblrIE(InfoExtractor):
|
||||
'url': 'https://www.tumblr.com/anyaboz/765332564457209856/my-music-video-for-selkie-by-nobodys-wolf-child',
|
||||
'info_dict': {
|
||||
'id': '765332564457209856',
|
||||
'timestamp': 1729878010,
|
||||
'upload_date': '20241025',
|
||||
'uploader_id': 'anyaboz',
|
||||
'repost_count': int,
|
||||
'age_limit': 0,
|
||||
@@ -445,6 +457,8 @@ class TumblrIE(InfoExtractor):
|
||||
'uploader_id': uploader_id,
|
||||
'uploader_url': f'https://{uploader_id}.tumblr.com/' if uploader_id else None,
|
||||
**traverse_obj(post_json, {
|
||||
# Try oldest post in reblog chain, fall back to timestamp of the post itself
|
||||
'timestamp': ((('trail', 0, 'post'), None), 'timestamp', {int_or_none}, any),
|
||||
'like_count': ('like_count', {int_or_none}),
|
||||
'repost_count': ('reblog_count', {int_or_none}),
|
||||
'tags': ('tags', ..., {str}),
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
int_or_none,
|
||||
join_nonempty,
|
||||
make_archive_id,
|
||||
parse_age_limit,
|
||||
smuggle_url,
|
||||
try_get,
|
||||
remove_end,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class TV5UnisBaseIE(InfoExtractor):
|
||||
_GEO_COUNTRIES = ['CA']
|
||||
_GEO_BYPASS = False
|
||||
|
||||
def _real_extract(self, url):
|
||||
groups = self._match_valid_url(url).groups()
|
||||
@@ -16,96 +20,136 @@ class TV5UnisBaseIE(InfoExtractor):
|
||||
'https://api.tv5unis.ca/graphql', groups[0], query={
|
||||
'query': '''{
|
||||
%s(%s) {
|
||||
title
|
||||
summary
|
||||
tags
|
||||
duration
|
||||
seasonNumber
|
||||
episodeNumber
|
||||
collection {
|
||||
title
|
||||
}
|
||||
episodeNumber
|
||||
rating {
|
||||
name
|
||||
}
|
||||
seasonNumber
|
||||
tags
|
||||
title
|
||||
videoElement {
|
||||
__typename
|
||||
... on Video {
|
||||
mediaId
|
||||
encodings {
|
||||
hls {
|
||||
url
|
||||
}
|
||||
}
|
||||
}
|
||||
... on RestrictedVideo {
|
||||
code
|
||||
reason
|
||||
}
|
||||
}
|
||||
}
|
||||
}''' % (self._GQL_QUERY_NAME, self._gql_args(groups)), # noqa: UP031
|
||||
})['data'][self._GQL_QUERY_NAME]
|
||||
media_id = product['videoElement']['mediaId']
|
||||
|
||||
video = product['videoElement']
|
||||
if video is None:
|
||||
raise ExtractorError('This content is no longer available', expected=True)
|
||||
|
||||
if video.get('__typename') == 'RestrictedVideo':
|
||||
code = video.get('code')
|
||||
if code == 1001:
|
||||
self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
|
||||
reason = video.get('reason')
|
||||
raise ExtractorError(join_nonempty(
|
||||
'This video is restricted',
|
||||
code is not None and f', error code {code}',
|
||||
reason and f': {remove_end(reason, ".")}',
|
||||
delim=''))
|
||||
|
||||
media_id = video['mediaId']
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||
video['encodings']['hls']['url'], media_id, 'mp4')
|
||||
|
||||
return {
|
||||
'_type': 'url_transparent',
|
||||
'id': media_id,
|
||||
'title': product.get('title'),
|
||||
'url': smuggle_url('limelight:media:' + media_id, {'geo_countries': self._GEO_COUNTRIES}),
|
||||
'age_limit': parse_age_limit(try_get(product, lambda x: x['rating']['name'])),
|
||||
'tags': product.get('tags'),
|
||||
'series': try_get(product, lambda x: x['collection']['title']),
|
||||
'season_number': int_or_none(product.get('seasonNumber')),
|
||||
'episode_number': int_or_none(product.get('episodeNumber')),
|
||||
'ie_key': 'LimelightMedia',
|
||||
'_old_archive_ids': [make_archive_id('LimelightMedia', media_id)],
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
**traverse_obj(product, {
|
||||
'title': ('title', {str}),
|
||||
'description': ('summary', {str}),
|
||||
'tags': ('tags', ..., {str}),
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'season_number': ('seasonNumber', {int_or_none}),
|
||||
'episode_number': ('episodeNumber', {int_or_none}),
|
||||
'series': ('collection', 'title', {str}),
|
||||
'age_limit': ('rating', 'name', {parse_age_limit}),
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
class TV5UnisVideoIE(TV5UnisBaseIE):
|
||||
_WORKING = False
|
||||
IE_NAME = 'tv5unis:video'
|
||||
_VALID_URL = r'https?://(?:www\.)?tv5unis\.ca/videos/[^/]+/(?P<id>\d+)'
|
||||
_TEST = {
|
||||
'url': 'https://www.tv5unis.ca/videos/bande-annonces/71843',
|
||||
'md5': '3d794164928bda97fb87a17e89923d9b',
|
||||
_VALID_URL = r'https?://(?:www\.)?tv5unis\.ca/videos/[^/?#]+/(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.tv5unis.ca/videos/bande-annonces/144041',
|
||||
'md5': '24a247c96119d77fe1bae8b440457dfa',
|
||||
'info_dict': {
|
||||
'id': 'a883684aecb2486cad9bdc7bbe17f861',
|
||||
'id': '56862325352147149dce0ae139afced6',
|
||||
'_old_archive_ids': ['limelightmedia 56862325352147149dce0ae139afced6'],
|
||||
'ext': 'mp4',
|
||||
'title': 'Watatatow',
|
||||
'duration': 10.01,
|
||||
'title': 'Antigone',
|
||||
'description': r"re:En aidant son frère .+ dicté par l'amour et la solidarité.",
|
||||
'duration': 61,
|
||||
},
|
||||
}
|
||||
}]
|
||||
_GQL_QUERY_NAME = 'productById'
|
||||
|
||||
@staticmethod
|
||||
def _gql_args(groups):
|
||||
return f'id: {groups}'
|
||||
return f'id: {groups[0]}'
|
||||
|
||||
|
||||
class TV5UnisIE(TV5UnisBaseIE):
|
||||
_WORKING = False
|
||||
IE_NAME = 'tv5unis'
|
||||
_VALID_URL = r'https?://(?:www\.)?tv5unis\.ca/videos/(?P<id>[^/]+)(?:/saisons/(?P<season_number>\d+)/episodes/(?P<episode_number>\d+))?/?(?:[?#&]|$)'
|
||||
_VALID_URL = r'https?://(?:www\.)?tv5unis\.ca/videos/(?P<id>[^/?#]+)(?:/saisons/(?P<season_number>\d+)/episodes/(?P<episode_number>\d+))?/?(?:[?#&]|$)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.tv5unis.ca/videos/watatatow/saisons/6/episodes/1',
|
||||
'md5': 'a479907d2e531a73e1f8dc48d6388d02',
|
||||
# geo-restricted to Canada; xff is ineffective
|
||||
'url': 'https://www.tv5unis.ca/videos/watatatow/saisons/11/episodes/1',
|
||||
'md5': '43beebd47eefb1c5caf9a47a3fc35589',
|
||||
'info_dict': {
|
||||
'id': 'e5ee23a586c44612a56aad61accf16ef',
|
||||
'id': '2c06e4af20f0417b86c2536825287690',
|
||||
'_old_archive_ids': ['limelightmedia 2c06e4af20f0417b86c2536825287690'],
|
||||
'ext': 'mp4',
|
||||
'title': 'Je ne peux pas lui résister',
|
||||
'description': "Atys, le nouveau concierge de l'école, a réussi à ébranler la confiance de Mado en affirmant qu'une médaille, ce n'est que du métal. Comme Mado essaie de lui prouver que ses valeurs sont solides, il veut la mettre à l'épreuve...",
|
||||
'title': "L'homme éléphant",
|
||||
'description': r're:Paul-André et Jean-Yves, .+ quand elle parle du feu au Spot.',
|
||||
'subtitles': {
|
||||
'fr': 'count:1',
|
||||
},
|
||||
'duration': 1370,
|
||||
'duration': 1440,
|
||||
'age_limit': 8,
|
||||
'tags': 'count:3',
|
||||
'tags': 'count:4',
|
||||
'series': 'Watatatow',
|
||||
'season_number': 6,
|
||||
'season': 'Season 11',
|
||||
'season_number': 11,
|
||||
'episode': 'Episode 1',
|
||||
'episode_number': 1,
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.tv5unis.ca/videos/le-voyage-de-fanny',
|
||||
'md5': '9ca80ebb575c681d10cae1adff3d4774',
|
||||
# geo-restricted to Canada; xff is ineffective
|
||||
'url': 'https://www.tv5unis.ca/videos/boite-a-savon',
|
||||
'md5': '7898e868e8c540f03844660e0aab6bbe',
|
||||
'info_dict': {
|
||||
'id': '726188eefe094d8faefb13381d42bc06',
|
||||
'id': '4de6d0c6467b4511a0c04b92037a9f15',
|
||||
'_old_archive_ids': ['limelightmedia 4de6d0c6467b4511a0c04b92037a9f15'],
|
||||
'ext': 'mp4',
|
||||
'title': 'Le voyage de Fanny',
|
||||
'description': "Fanny, 12 ans, cachée dans un foyer loin de ses parents, s'occupe de ses deux soeurs. Devant fuir, Fanny prend la tête d'un groupe de huit enfants et s'engage dans un dangereux périple à travers la France occupée pour rejoindre la frontière suisse.",
|
||||
'title': 'Boîte à savon',
|
||||
'description': r're:Dans le petit village de Broche-à-foin, .+ celle qui fait battre son coeur.',
|
||||
'subtitles': {
|
||||
'fr': 'count:1',
|
||||
},
|
||||
'duration': 5587.034,
|
||||
'tags': 'count:4',
|
||||
'duration': 1200,
|
||||
'tags': 'count:5',
|
||||
},
|
||||
}]
|
||||
_GQL_QUERY_NAME = 'productByRootProductSlug'
|
||||
|
||||
@@ -41,16 +41,16 @@ class TwitchBaseIE(InfoExtractor):
|
||||
_NETRC_MACHINE = 'twitch'
|
||||
|
||||
_OPERATION_HASHES = {
|
||||
'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14',
|
||||
'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb',
|
||||
'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777',
|
||||
'ShareClipRenderStatus': 'e0a46b287d760c6890a39d1ccd736af5ec9479a267d02c710e9ac33326b651d2',
|
||||
'ChannelCollectionsContent': '447aec6a0cc1e8d0a8d7732d47eb0762c336a2294fdb009e9c9d854e49d484b9',
|
||||
'StreamMetadata': 'a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962',
|
||||
'CollectionSideBar': '016e1e4ccee0eb4698eb3bf1a04dc1c077fb746c78c82bac9a8f0289658fbd1a',
|
||||
'FilterableVideoTower_Videos': '67004f7881e65c297936f32c75246470629557a393788fb5a69d6d9a25a8fd5f',
|
||||
'ClipsCards__User': '90c33f5e6465122fba8f9371e2a97076f9ed06c6fed3788d002ab9eba8f91d88',
|
||||
'ShareClipRenderStatus': '1844261bb449fa51e6167040311da4a7a5f1c34fe71c71a3e0c4f551bc30c698',
|
||||
'ChannelCollectionsContent': '5247910a19b1cd2b760939bf4cba4dcbd3d13bdf8c266decd16956f6ef814077',
|
||||
'StreamMetadata': 'b57f9b910f8cd1a4659d894fe7550ccc81ec9052c01e438b290fd66a040b9b93',
|
||||
'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
|
||||
'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
|
||||
'VideoMetadata': '49b5b8f268cdeb259d75b58dcb0c1a748e3b575003448a2333dc5cdafd49adad',
|
||||
'VideoPlayer_ChapterSelectButtonVideo': '8d2793384aac3773beab5e59bd5d6f585aedb923d292800119e03d40cd0f9b41',
|
||||
'VideoPreviewOverlay': '9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f',
|
||||
'VideoMetadata': '45111672eea2e507f8ba44d101a61862f9c56b11dee09a15634cb75cb9b9084d',
|
||||
'VideoPlayer_ChapterSelectButtonVideo': '71835d5ef425e154bf282453a926d99b328cdc5e32f36d3a209d0f4778b41203',
|
||||
'VideoPlayer_VODSeekbarPreviewVideo': '07e99e4d56c5a7c67117a154777b0baf85a5ffefa393b213f4bc712ccaf85dd6',
|
||||
}
|
||||
|
||||
@@ -621,15 +621,15 @@ def _make_video_result(node):
|
||||
|
||||
|
||||
class TwitchCollectionIE(TwitchBaseIE):
|
||||
IE_NAME = 'twitch:collection'
|
||||
_VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/collections/(?P<id>[^/]+)'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://www.twitch.tv/collections/wlDCoH0zEBZZbQ',
|
||||
'url': 'https://www.twitch.tv/collections/o9zZer3IQBhTJw',
|
||||
'info_dict': {
|
||||
'id': 'wlDCoH0zEBZZbQ',
|
||||
'title': 'Overthrow Nook, capitalism for children',
|
||||
'id': 'o9zZer3IQBhTJw',
|
||||
'title': 'Playthrough Archives',
|
||||
},
|
||||
'playlist_mincount': 13,
|
||||
'playlist_mincount': 21,
|
||||
}]
|
||||
|
||||
_OPERATION_NAME = 'CollectionSideBar'
|
||||
@@ -680,6 +680,10 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
|
||||
}],
|
||||
f'Downloading {self._NODE_KIND}s GraphQL page {page_num}',
|
||||
fatal=False)
|
||||
# Avoid extracting random/unrelated entries when channel_name doesn't exist
|
||||
# See https://github.com/yt-dlp/yt-dlp/issues/15450
|
||||
if traverse_obj(page, (0, 'data', 'user', 'id', {str})) == '':
|
||||
raise ExtractorError(f'Channel "{channel_name}" not found', expected=True)
|
||||
if not page:
|
||||
break
|
||||
edges = try_get(
|
||||
@@ -720,8 +724,8 @@ class TwitchVideosBaseIE(TwitchPlaylistBaseIE):
|
||||
|
||||
|
||||
class TwitchVideosIE(TwitchVideosBaseIE):
|
||||
IE_NAME = 'twitch:videos'
|
||||
_VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/(?:videos|profile)'
|
||||
|
||||
_TESTS = [{
|
||||
# All Videos sorted by Date
|
||||
'url': 'https://www.twitch.tv/spamfish/videos?filter=all',
|
||||
@@ -729,7 +733,7 @@ class TwitchVideosIE(TwitchVideosBaseIE):
|
||||
'id': 'spamfish',
|
||||
'title': 'spamfish - All Videos sorted by Date',
|
||||
},
|
||||
'playlist_mincount': 924,
|
||||
'playlist_mincount': 751,
|
||||
}, {
|
||||
# All Videos sorted by Popular
|
||||
'url': 'https://www.twitch.tv/spamfish/videos?filter=all&sort=views',
|
||||
@@ -737,8 +741,9 @@ class TwitchVideosIE(TwitchVideosBaseIE):
|
||||
'id': 'spamfish',
|
||||
'title': 'spamfish - All Videos sorted by Popular',
|
||||
},
|
||||
'playlist_mincount': 931,
|
||||
'playlist_mincount': 754,
|
||||
}, {
|
||||
# TODO: Investigate why we get 0 entries
|
||||
# Past Broadcasts sorted by Date
|
||||
'url': 'https://www.twitch.tv/spamfish/videos?filter=archives',
|
||||
'info_dict': {
|
||||
@@ -753,8 +758,9 @@ class TwitchVideosIE(TwitchVideosBaseIE):
|
||||
'id': 'spamfish',
|
||||
'title': 'spamfish - Highlights sorted by Date',
|
||||
},
|
||||
'playlist_mincount': 901,
|
||||
'playlist_mincount': 751,
|
||||
}, {
|
||||
# TODO: Investigate why we get 0 entries
|
||||
# Uploads sorted by Date
|
||||
'url': 'https://www.twitch.tv/esl_csgo/videos?filter=uploads&sort=time',
|
||||
'info_dict': {
|
||||
@@ -763,6 +769,7 @@ class TwitchVideosIE(TwitchVideosBaseIE):
|
||||
},
|
||||
'playlist_mincount': 5,
|
||||
}, {
|
||||
# TODO: Investigate why we get 0 entries
|
||||
# Past Premieres sorted by Date
|
||||
'url': 'https://www.twitch.tv/spamfish/videos?filter=past_premieres',
|
||||
'info_dict': {
|
||||
@@ -825,8 +832,8 @@ class TwitchVideosIE(TwitchVideosBaseIE):
|
||||
|
||||
|
||||
class TwitchVideosClipsIE(TwitchPlaylistBaseIE):
|
||||
IE_NAME = 'twitch:videos:clips'
|
||||
_VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/(?:clips|videos/*?\?.*?\bfilter=clips)'
|
||||
|
||||
_TESTS = [{
|
||||
# Clips
|
||||
'url': 'https://www.twitch.tv/vanillatv/clips?filter=clips&range=all',
|
||||
@@ -898,8 +905,8 @@ class TwitchVideosClipsIE(TwitchPlaylistBaseIE):
|
||||
|
||||
|
||||
class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE):
|
||||
IE_NAME = 'twitch:videos:collections'
|
||||
_VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/videos/*?\?.*?\bfilter=collections'
|
||||
|
||||
_TESTS = [{
|
||||
# Collections
|
||||
'url': 'https://www.twitch.tv/spamfish/videos?filter=collections',
|
||||
@@ -1050,7 +1057,10 @@ class TwitchStreamIE(TwitchVideosBaseIE):
|
||||
gql = self._download_gql(
|
||||
channel_name, [{
|
||||
'operationName': 'StreamMetadata',
|
||||
'variables': {'channelLogin': channel_name},
|
||||
'variables': {
|
||||
'channelLogin': channel_name,
|
||||
'includeIsDJ': True,
|
||||
},
|
||||
}, {
|
||||
'operationName': 'ComscoreStreamingQuery',
|
||||
'variables': {
|
||||
|
||||
@@ -32,67 +32,11 @@ from ..utils.traversal import require, traverse_obj
|
||||
|
||||
|
||||
class TwitterBaseIE(InfoExtractor):
|
||||
_NETRC_MACHINE = 'twitter'
|
||||
_API_BASE = 'https://api.x.com/1.1/'
|
||||
_GRAPHQL_API_BASE = 'https://x.com/i/api/graphql/'
|
||||
_BASE_REGEX = r'https?://(?:(?:www|m(?:obile)?)\.)?(?:(?:twitter|x)\.com|twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid\.onion)/'
|
||||
_AUTH = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
|
||||
_LEGACY_AUTH = 'AAAAAAAAAAAAAAAAAAAAAIK1zgAAAAAA2tUWuhGZ2JceoId5GwYWU5GspY4%3DUq7gzFoCZs1QfwGoVdvSac3IniczZEYXIcDyumCauIXpcAPorE'
|
||||
_flow_token = None
|
||||
|
||||
_LOGIN_INIT_DATA = json.dumps({
|
||||
'input_flow_data': {
|
||||
'flow_context': {
|
||||
'debug_overrides': {},
|
||||
'start_location': {
|
||||
'location': 'unknown',
|
||||
},
|
||||
},
|
||||
},
|
||||
'subtask_versions': {
|
||||
'action_list': 2,
|
||||
'alert_dialog': 1,
|
||||
'app_download_cta': 1,
|
||||
'check_logged_in_account': 1,
|
||||
'choice_selection': 3,
|
||||
'contacts_live_sync_permission_prompt': 0,
|
||||
'cta': 7,
|
||||
'email_verification': 2,
|
||||
'end_flow': 1,
|
||||
'enter_date': 1,
|
||||
'enter_email': 2,
|
||||
'enter_password': 5,
|
||||
'enter_phone': 2,
|
||||
'enter_recaptcha': 1,
|
||||
'enter_text': 5,
|
||||
'enter_username': 2,
|
||||
'generic_urt': 3,
|
||||
'in_app_notification': 1,
|
||||
'interest_picker': 3,
|
||||
'js_instrumentation': 1,
|
||||
'menu_dialog': 1,
|
||||
'notifications_permission_prompt': 2,
|
||||
'open_account': 2,
|
||||
'open_home_timeline': 1,
|
||||
'open_link': 1,
|
||||
'phone_verification': 4,
|
||||
'privacy_options': 1,
|
||||
'security_key': 3,
|
||||
'select_avatar': 4,
|
||||
'select_banner': 2,
|
||||
'settings_list': 7,
|
||||
'show_code': 1,
|
||||
'sign_up': 2,
|
||||
'sign_up_review': 4,
|
||||
'tweet_selection_urt': 1,
|
||||
'update_users': 1,
|
||||
'upload_media': 1,
|
||||
'user_recommendations_list': 4,
|
||||
'user_recommendations_urt': 1,
|
||||
'wait_spinner': 3,
|
||||
'web_modal': 1,
|
||||
},
|
||||
}, separators=(',', ':')).encode()
|
||||
|
||||
def _extract_variant_formats(self, variant, video_id):
|
||||
variant_url = variant.get('url')
|
||||
@@ -172,135 +116,6 @@ class TwitterBaseIE(InfoExtractor):
|
||||
'x-csrf-token': try_call(lambda: self._get_cookies(self._API_BASE)['ct0'].value),
|
||||
})
|
||||
|
||||
def _call_login_api(self, note, headers, query={}, data=None):
|
||||
response = self._download_json(
|
||||
f'{self._API_BASE}onboarding/task.json', None, note,
|
||||
headers=headers, query=query, data=data, expected_status=400)
|
||||
error = traverse_obj(response, ('errors', 0, 'message', {str}))
|
||||
if error:
|
||||
raise ExtractorError(f'Login failed, Twitter API says: {error}', expected=True)
|
||||
elif traverse_obj(response, 'status') != 'success':
|
||||
raise ExtractorError('Login was unsuccessful')
|
||||
|
||||
subtask = traverse_obj(
|
||||
response, ('subtasks', ..., 'subtask_id', {str}), get_all=False)
|
||||
if not subtask:
|
||||
raise ExtractorError('Twitter API did not return next login subtask')
|
||||
|
||||
self._flow_token = response['flow_token']
|
||||
|
||||
return subtask
|
||||
|
||||
def _perform_login(self, username, password):
|
||||
if self.is_logged_in:
|
||||
return
|
||||
|
||||
guest_token = self._fetch_guest_token(None)
|
||||
headers = {
|
||||
**self._set_base_headers(),
|
||||
'content-type': 'application/json',
|
||||
'x-guest-token': guest_token,
|
||||
'x-twitter-client-language': 'en',
|
||||
'x-twitter-active-user': 'yes',
|
||||
'Referer': 'https://x.com/',
|
||||
'Origin': 'https://x.com',
|
||||
}
|
||||
|
||||
def build_login_json(*subtask_inputs):
|
||||
return json.dumps({
|
||||
'flow_token': self._flow_token,
|
||||
'subtask_inputs': subtask_inputs,
|
||||
}, separators=(',', ':')).encode()
|
||||
|
||||
def input_dict(subtask_id, text):
|
||||
return {
|
||||
'subtask_id': subtask_id,
|
||||
'enter_text': {
|
||||
'text': text,
|
||||
'link': 'next_link',
|
||||
},
|
||||
}
|
||||
|
||||
next_subtask = self._call_login_api(
|
||||
'Downloading flow token', headers, query={'flow_name': 'login'}, data=self._LOGIN_INIT_DATA)
|
||||
|
||||
while not self.is_logged_in:
|
||||
if next_subtask == 'LoginJsInstrumentationSubtask':
|
||||
next_subtask = self._call_login_api(
|
||||
'Submitting JS instrumentation response', headers, data=build_login_json({
|
||||
'subtask_id': next_subtask,
|
||||
'js_instrumentation': {
|
||||
'response': '{}',
|
||||
'link': 'next_link',
|
||||
},
|
||||
}))
|
||||
|
||||
elif next_subtask == 'LoginEnterUserIdentifierSSO':
|
||||
next_subtask = self._call_login_api(
|
||||
'Submitting username', headers, data=build_login_json({
|
||||
'subtask_id': next_subtask,
|
||||
'settings_list': {
|
||||
'setting_responses': [{
|
||||
'key': 'user_identifier',
|
||||
'response_data': {
|
||||
'text_data': {
|
||||
'result': username,
|
||||
},
|
||||
},
|
||||
}],
|
||||
'link': 'next_link',
|
||||
},
|
||||
}))
|
||||
|
||||
elif next_subtask == 'LoginEnterAlternateIdentifierSubtask':
|
||||
next_subtask = self._call_login_api(
|
||||
'Submitting alternate identifier', headers,
|
||||
data=build_login_json(input_dict(next_subtask, self._get_tfa_info(
|
||||
'one of username, phone number or email that was not used as --username'))))
|
||||
|
||||
elif next_subtask == 'LoginEnterPassword':
|
||||
next_subtask = self._call_login_api(
|
||||
'Submitting password', headers, data=build_login_json({
|
||||
'subtask_id': next_subtask,
|
||||
'enter_password': {
|
||||
'password': password,
|
||||
'link': 'next_link',
|
||||
},
|
||||
}))
|
||||
|
||||
elif next_subtask == 'AccountDuplicationCheck':
|
||||
next_subtask = self._call_login_api(
|
||||
'Submitting account duplication check', headers, data=build_login_json({
|
||||
'subtask_id': next_subtask,
|
||||
'check_logged_in_account': {
|
||||
'link': 'AccountDuplicationCheck_false',
|
||||
},
|
||||
}))
|
||||
|
||||
elif next_subtask == 'LoginTwoFactorAuthChallenge':
|
||||
next_subtask = self._call_login_api(
|
||||
'Submitting 2FA token', headers, data=build_login_json(input_dict(
|
||||
next_subtask, self._get_tfa_info('two-factor authentication token'))))
|
||||
|
||||
elif next_subtask == 'LoginAcid':
|
||||
next_subtask = self._call_login_api(
|
||||
'Submitting confirmation code', headers, data=build_login_json(input_dict(
|
||||
next_subtask, self._get_tfa_info('confirmation code sent to your email or phone'))))
|
||||
|
||||
elif next_subtask == 'ArkoseLogin':
|
||||
self.raise_login_required('Twitter is requiring captcha for this login attempt', method='cookies')
|
||||
|
||||
elif next_subtask == 'DenyLoginSubtask':
|
||||
self.raise_login_required('Twitter rejected this login attempt as suspicious', method='cookies')
|
||||
|
||||
elif next_subtask == 'LoginSuccessSubtask':
|
||||
raise ExtractorError('Twitter API did not grant auth token cookie')
|
||||
|
||||
else:
|
||||
raise ExtractorError(f'Unrecognized subtask ID "{next_subtask}"')
|
||||
|
||||
self.report_login()
|
||||
|
||||
def _call_api(self, path, video_id, query={}, graphql=False):
|
||||
headers = self._set_base_headers(legacy=not graphql and self._selected_api == 'legacy')
|
||||
headers.update({
|
||||
@@ -416,6 +231,7 @@ class TwitterCardIE(InfoExtractor):
|
||||
'live_status': 'not_live',
|
||||
},
|
||||
'add_ie': ['Youtube'],
|
||||
'skip': 'The page does not exist',
|
||||
},
|
||||
{
|
||||
'url': 'https://twitter.com/i/videos/tweet/705235433198714880',
|
||||
@@ -617,6 +433,7 @@ class TwitterIE(TwitterBaseIE):
|
||||
'comment_count': int,
|
||||
'_old_archive_ids': ['twitter 852138619213144067'],
|
||||
},
|
||||
'skip': 'Suspended',
|
||||
}, {
|
||||
'url': 'https://twitter.com/i/web/status/910031516746514432',
|
||||
'info_dict': {
|
||||
@@ -763,10 +580,10 @@ class TwitterIE(TwitterBaseIE):
|
||||
'url': 'https://twitter.com/UltimaShadowX/status/1577719286659006464',
|
||||
'info_dict': {
|
||||
'id': '1577719286659006464',
|
||||
'title': 'Ultima - Test',
|
||||
'title': r're:Ultima.* - Test$',
|
||||
'description': 'Test https://t.co/Y3KEZD7Dad',
|
||||
'channel_id': '168922496',
|
||||
'uploader': 'Ultima',
|
||||
'uploader': r're:Ultima.*',
|
||||
'uploader_id': 'UltimaShadowX',
|
||||
'uploader_url': 'https://twitter.com/UltimaShadowX',
|
||||
'upload_date': '20221005',
|
||||
@@ -895,11 +712,12 @@ class TwitterIE(TwitterBaseIE):
|
||||
'uploader': r're:Monique Camarra.+?',
|
||||
'uploader_id': 'MoniqueCamarra',
|
||||
'live_status': 'was_live',
|
||||
'release_timestamp': 1658417414,
|
||||
'release_timestamp': 1658417305,
|
||||
'description': r're:Twitter Space participated by Sergej Sumlenny.+',
|
||||
'timestamp': 1658407771,
|
||||
'release_date': '20220721',
|
||||
'upload_date': '20220721',
|
||||
'thumbnail': 'https://pbs.twimg.com/profile_images/1920514378006188033/xQs6J_yI_400x400.jpg',
|
||||
},
|
||||
'add_ie': ['TwitterSpaces'],
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
@@ -1010,10 +828,10 @@ class TwitterIE(TwitterBaseIE):
|
||||
'description': 'This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525 https://t.co/cNsA0MoOml',
|
||||
'thumbnail': 'https://pbs.twimg.com/ext_tw_video_thumb/1600009362759733248/pu/img/XVhFQivj75H_YxxV.jpg?name=orig',
|
||||
'age_limit': 0,
|
||||
'uploader': 'Boy Called Mün',
|
||||
'uploader': 'D U N I Y A',
|
||||
'repost_count': int,
|
||||
'upload_date': '20221206',
|
||||
'title': 'Boy Called Mün - This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525',
|
||||
'title': 'D U N I Y A - This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525',
|
||||
'comment_count': int,
|
||||
'like_count': int,
|
||||
'tags': [],
|
||||
@@ -1068,6 +886,7 @@ class TwitterIE(TwitterBaseIE):
|
||||
'comment_count': int,
|
||||
'_old_archive_ids': ['twitter 1695424220702888009'],
|
||||
},
|
||||
'skip': 'Suspended',
|
||||
}, {
|
||||
# retweeted_status w/ legacy API
|
||||
'url': 'https://twitter.com/playstrumpcard/status/1695424220702888009',
|
||||
@@ -1092,6 +911,7 @@ class TwitterIE(TwitterBaseIE):
|
||||
'_old_archive_ids': ['twitter 1695424220702888009'],
|
||||
},
|
||||
'params': {'extractor_args': {'twitter': {'api': ['legacy']}}},
|
||||
'skip': 'Suspended',
|
||||
}, {
|
||||
# Broadcast embedded in tweet
|
||||
'url': 'https://twitter.com/JessicaDobsonWX/status/1731121063248175384',
|
||||
@@ -1135,7 +955,6 @@ class TwitterIE(TwitterBaseIE):
|
||||
}, {
|
||||
# "stale tweet" with typename "TweetWithVisibilityResults"
|
||||
'url': 'https://twitter.com/RobertKennedyJr/status/1724884212803834154',
|
||||
'md5': '511377ff8dfa7545307084dca4dce319',
|
||||
'info_dict': {
|
||||
'id': '1724883339285544960',
|
||||
'ext': 'mp4',
|
||||
@@ -1182,6 +1001,30 @@ class TwitterIE(TwitterBaseIE):
|
||||
'age_limit': 0,
|
||||
'_old_archive_ids': ['twitter 1790637656616943991'],
|
||||
},
|
||||
}, {
|
||||
# unified_card with 2 items of type video and photo
|
||||
'url': 'https://x.com/TopHeroes_/status/2001950365332455490',
|
||||
'info_dict': {
|
||||
'id': '2001841416071450628',
|
||||
'ext': 'mp4',
|
||||
'display_id': '2001950365332455490',
|
||||
'title': 'Top Heroes - Forgot to close My heroes solo level up in my phone ✨Unlock the fog,...',
|
||||
'description': r're:Forgot to close My heroes solo level up in my phone ✨Unlock the fog.+',
|
||||
'uploader': 'Top Heroes',
|
||||
'uploader_id': 'TopHeroes_',
|
||||
'uploader_url': 'https://twitter.com/TopHeroes_',
|
||||
'channel_id': '1737324725620326400',
|
||||
'comment_count': int,
|
||||
'like_count': int,
|
||||
'repost_count': int,
|
||||
'age_limit': 0,
|
||||
'duration': 30.278,
|
||||
'thumbnail': 'https://pbs.twimg.com/amplify_video_thumb/2001841416071450628/img/hpy5KpJh4pO17b65.jpg?name=orig',
|
||||
'tags': [],
|
||||
'timestamp': 1766137136,
|
||||
'upload_date': '20251219',
|
||||
'_old_archive_ids': ['twitter 2001950365332455490'],
|
||||
},
|
||||
}, {
|
||||
# onion route
|
||||
'url': 'https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/TwitterBlue/status/1484226494708662273',
|
||||
@@ -1422,14 +1265,14 @@ class TwitterIE(TwitterBaseIE):
|
||||
if not card:
|
||||
return
|
||||
|
||||
self.write_debug(f'Extracting from card info: {card.get("url")}')
|
||||
card_name = card['name'].split(':')[-1]
|
||||
self.write_debug(f'Extracting from {card_name} card info: {card.get("url")}')
|
||||
binding_values = card['binding_values']
|
||||
|
||||
def get_binding_value(k):
|
||||
o = binding_values.get(k) or {}
|
||||
return try_get(o, lambda x: x[x['type'].lower() + '_value'])
|
||||
|
||||
card_name = card['name'].split(':')[-1]
|
||||
if card_name == 'player':
|
||||
yield {
|
||||
'_type': 'url',
|
||||
@@ -1461,7 +1304,7 @@ class TwitterIE(TwitterBaseIE):
|
||||
elif card_name == 'unified_card':
|
||||
unified_card = self._parse_json(get_binding_value('unified_card'), twid)
|
||||
yield from map(extract_from_video_info, traverse_obj(
|
||||
unified_card, ('media_entities', ...), expected_type=dict))
|
||||
unified_card, ('media_entities', lambda _, v: v['type'] == 'video')))
|
||||
# amplify, promo_video_website, promo_video_convo, appplayer,
|
||||
# video_direct_message, poll2choice_video, poll3choice_video,
|
||||
# poll4choice_video, ...
|
||||
|
||||
@@ -7,14 +7,15 @@ from ..utils import (
|
||||
parse_age_limit,
|
||||
try_get,
|
||||
unified_timestamp,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import require, traverse_obj
|
||||
|
||||
|
||||
class URPlayIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?ur(?:play|skola)\.se/(?:program|Produkter)/(?P<id>[0-9]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://urplay.se/program/203704-ur-samtiden-livet-universum-och-rymdens-markliga-musik-om-vetenskap-kritiskt-tankande-och-motstand',
|
||||
'md5': '5ba36643c77cc3d34ffeadad89937d1e',
|
||||
'info_dict': {
|
||||
'id': '203704',
|
||||
'ext': 'mp4',
|
||||
@@ -25,11 +26,12 @@ class URPlayIE(InfoExtractor):
|
||||
'upload_date': '20171214',
|
||||
'series': 'UR Samtiden - Livet, universum och rymdens märkliga musik',
|
||||
'duration': 2269,
|
||||
'categories': ['Vetenskap & teknik'],
|
||||
'categories': ['Kultur & historia'],
|
||||
'tags': ['Kritiskt tänkande', 'Vetenskap', 'Vetenskaplig verksamhet'],
|
||||
'episode': 'Om vetenskap, kritiskt tänkande och motstånd',
|
||||
'age_limit': 15,
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}, {
|
||||
'url': 'https://urplay.se/program/222967-en-foralders-dagbok-mitt-barn-skadar-sig-sjalv',
|
||||
'info_dict': {
|
||||
@@ -48,6 +50,7 @@ class URPlayIE(InfoExtractor):
|
||||
'tags': 'count:7',
|
||||
'episode': 'Mitt barn skadar sig själv',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}, {
|
||||
'url': 'https://urskola.se/Produkter/190031-Tripp-Trapp-Trad-Sovkudde',
|
||||
'info_dict': {
|
||||
@@ -67,6 +70,27 @@ class URPlayIE(InfoExtractor):
|
||||
'episode': 'Sovkudde',
|
||||
'season': 'Säsong 1',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}, {
|
||||
# Only accessible through new media api
|
||||
'url': 'https://urplay.se/program/242932-vulkanernas-krafter-fran-kraftfull-till-forgorande',
|
||||
'info_dict': {
|
||||
'id': '242932',
|
||||
'ext': 'mp4',
|
||||
'title': 'Vulkanernas krafter : Från kraftfull till förgörande',
|
||||
'description': 'md5:742bb87048e7d5a7f209d28f9bb70ab1',
|
||||
'age_limit': 15,
|
||||
'duration': 2613,
|
||||
'thumbnail': 'https://assets.ur.se/id/242932/images/1_hd.jpg',
|
||||
'categories': ['Vetenskap & teknik'],
|
||||
'tags': ['Geofysik', 'Naturvetenskap', 'Vulkaner', 'Vulkanutbrott'],
|
||||
'series': 'Vulkanernas krafter',
|
||||
'episode': 'Från kraftfull till förgörande',
|
||||
'episode_number': 2,
|
||||
'timestamp': 1763514000,
|
||||
'upload_date': '20251119',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}, {
|
||||
'url': 'http://urskola.se/Produkter/155794-Smasagor-meankieli-Grodan-i-vida-varlden',
|
||||
'only_matching': True,
|
||||
@@ -78,7 +102,7 @@ class URPlayIE(InfoExtractor):
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
urplayer_data = self._search_nextjs_data(webpage, video_id, fatal=False) or {}
|
||||
if urplayer_data:
|
||||
urplayer_data = try_get(urplayer_data, lambda x: x['props']['pageProps']['program'], dict)
|
||||
urplayer_data = traverse_obj(urplayer_data, ('props', 'pageProps', 'productData', {dict}))
|
||||
if not urplayer_data:
|
||||
raise ExtractorError('Unable to parse __NEXT_DATA__')
|
||||
else:
|
||||
@@ -87,21 +111,12 @@ class URPlayIE(InfoExtractor):
|
||||
webpage, 'urplayer data'), video_id)['accessibleEpisodes']
|
||||
urplayer_data = next(e for e in accessible_episodes if e.get('id') == int_or_none(video_id))
|
||||
episode = urplayer_data['title']
|
||||
|
||||
host = self._download_json('http://streaming-loadbalancer.ur.se/loadbalancer.json', video_id)['redirect']
|
||||
formats = []
|
||||
urplayer_streams = urplayer_data.get('streamingInfo', {})
|
||||
|
||||
for k, v in urplayer_streams.get('raw', {}).items():
|
||||
if not (k in ('sd', 'hd', 'mp3', 'm4a') and isinstance(v, dict)):
|
||||
continue
|
||||
file_http = v.get('location')
|
||||
if file_http:
|
||||
formats.extend(self._extract_wowza_formats(
|
||||
f'http://{host}/{file_http}playlist.m3u8',
|
||||
video_id, skip_protocols=['f4m', 'rtmp', 'rtsp']))
|
||||
|
||||
subtitles = {}
|
||||
sources = self._download_json(
|
||||
f'https://media-api.urplay.se/config-streaming/v1/urplay/sources/{video_id}', video_id,
|
||||
note='Downloading streaming information')
|
||||
hls_url = traverse_obj(sources, ('sources', 'hls', {url_or_none}, {require('HLS URL')}))
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||
hls_url, video_id, 'mp4', m3u8_id='hls')
|
||||
|
||||
def parse_lang_code(code):
|
||||
"3-character language code or None (utils candidate)"
|
||||
|
||||
@@ -339,11 +339,20 @@ class WistiaChannelIE(WistiaBaseIE):
|
||||
'title': 'The Roof S2: The Modern CRO',
|
||||
'thumbnail': r're:https?://embed(?:-ssl)?\.wistia\.com/.+\.(?:jpg|png)',
|
||||
'duration': 86.487,
|
||||
'description': 'A sales leader on The Roof? Man, they really must be letting anyone up here this season.\n',
|
||||
'description': 'A sales leader on The Roof? Man, they really must be letting anyone up here this season. ',
|
||||
'timestamp': 1619790290,
|
||||
'upload_date': '20210430',
|
||||
},
|
||||
'params': {'noplaylist': True, 'skip_download': True},
|
||||
}, {
|
||||
# Channel with episodes structure instead of videos
|
||||
'url': 'https://fast.wistia.net/embed/channel/sapab9p6qd',
|
||||
'info_dict': {
|
||||
'id': 'sapab9p6qd',
|
||||
'title': 'Credo: An RCIA Program',
|
||||
'description': '\n',
|
||||
},
|
||||
'playlist_mincount': 80,
|
||||
}]
|
||||
_WEBPAGE_TESTS = [{
|
||||
'url': 'https://www.profitwell.com/recur/boxed-out',
|
||||
@@ -399,8 +408,7 @@ class WistiaChannelIE(WistiaBaseIE):
|
||||
|
||||
entries = [
|
||||
self.url_result(f'wistia:{video["hashedId"]}', WistiaIE, title=video.get('name'))
|
||||
for video in traverse_obj(series, ('sections', ..., 'videos', ...)) or []
|
||||
if video.get('hashedId')
|
||||
for video in traverse_obj(series, ('sections', ..., ('videos', 'episodes'), lambda _, v: v['hashedId']))
|
||||
]
|
||||
|
||||
return self.playlist_result(
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import base64
|
||||
import codecs
|
||||
import itertools
|
||||
import re
|
||||
import string
|
||||
import urllib.parse
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
@@ -16,7 +14,6 @@ from ..utils import (
|
||||
join_nonempty,
|
||||
parse_duration,
|
||||
str_or_none,
|
||||
try_call,
|
||||
try_get,
|
||||
unified_strdate,
|
||||
url_or_none,
|
||||
@@ -32,7 +29,7 @@ class _ByteGenerator:
|
||||
try:
|
||||
self._algorithm = getattr(self, f'_algo{algo_id}')
|
||||
except AttributeError:
|
||||
raise ExtractorError(f'Unknown algorithm ID: {algo_id}')
|
||||
raise ExtractorError(f'Unknown algorithm ID "{algo_id}"')
|
||||
self._s = to_signed_32(seed)
|
||||
|
||||
def _algo1(self, s):
|
||||
@@ -60,6 +57,37 @@ class _ByteGenerator:
|
||||
s = to_signed_32(s * to_signed_32(0xc2b2ae3d))
|
||||
return to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 16))
|
||||
|
||||
def _algo4(self, s):
|
||||
# Custom scrambling function involving a left rotation (ROL)
|
||||
s = self._s = to_signed_32(s + 0x6d2b79f5)
|
||||
s = to_signed_32((s << 7) | ((s & 0xFFFFFFFF) >> 25)) # ROL 7
|
||||
s = to_signed_32(s + 0x9e3779b9)
|
||||
s = to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 11))
|
||||
return to_signed_32(s * 0x27d4eb2d)
|
||||
|
||||
def _algo5(self, s):
|
||||
# xorshift variant with a final addition
|
||||
s = to_signed_32(s ^ (s << 7))
|
||||
s = to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 9))
|
||||
s = to_signed_32(s ^ (s << 8))
|
||||
s = self._s = to_signed_32(s + 0xa5a5a5a5)
|
||||
return s
|
||||
|
||||
def _algo6(self, s):
|
||||
# LCG (a=0x2c9277b5, c=0xac564b05) with a variable right shift scrambler
|
||||
s = self._s = to_signed_32(s * to_signed_32(0x2c9277b5) + to_signed_32(0xac564b05))
|
||||
s2 = to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 18))
|
||||
shift = (s & 0xFFFFFFFF) >> 27 & 31
|
||||
return to_signed_32((s2 & 0xFFFFFFFF) >> shift)
|
||||
|
||||
def _algo7(self, s):
|
||||
# Weyl Sequence (k=0x9e3779b9) + custom multiply-xor-shift mixing function
|
||||
s = self._s = to_signed_32(s + to_signed_32(0x9e3779b9))
|
||||
e = to_signed_32(s ^ (s << 5))
|
||||
e = to_signed_32(e * to_signed_32(0x7feb352d))
|
||||
e = to_signed_32(e ^ ((e & 0xFFFFFFFF) >> 15))
|
||||
return to_signed_32(e * to_signed_32(0x846ca68b))
|
||||
|
||||
def __next__(self):
|
||||
return self._algorithm(self._s) & 0xFF
|
||||
|
||||
@@ -185,32 +213,28 @@ class XHamsterIE(InfoExtractor):
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
_XOR_KEY = b'xh7999'
|
||||
|
||||
def _decipher_format_url(self, format_url, format_id):
|
||||
if all(char in string.hexdigits for char in format_url):
|
||||
byte_data = bytes.fromhex(format_url)
|
||||
seed = int.from_bytes(byte_data[1:5], byteorder='little', signed=True)
|
||||
byte_gen = _ByteGenerator(byte_data[0], seed)
|
||||
return bytearray(byte ^ next(byte_gen) for byte in byte_data[5:]).decode('latin-1')
|
||||
parsed_url = urllib.parse.urlparse(format_url)
|
||||
|
||||
cipher_type, _, ciphertext = try_call(
|
||||
lambda: base64.b64decode(format_url).decode().partition('_')) or [None] * 3
|
||||
|
||||
if not cipher_type or not ciphertext:
|
||||
self.report_warning(f'Skipping format "{format_id}": failed to decipher URL')
|
||||
hex_string, path_remainder = self._search_regex(
|
||||
r'^/(?P<hex>[0-9a-fA-F]{12,})(?P<rem>[/,].+)$', parsed_url.path, 'url components',
|
||||
default=(None, None), group=('hex', 'rem'))
|
||||
if not hex_string:
|
||||
self.report_warning(f'Skipping format "{format_id}": unsupported URL format')
|
||||
return None
|
||||
|
||||
if cipher_type == 'xor':
|
||||
return bytes(
|
||||
a ^ b for a, b in
|
||||
zip(ciphertext.encode(), itertools.cycle(self._XOR_KEY))).decode()
|
||||
byte_data = bytes.fromhex(hex_string)
|
||||
seed = int.from_bytes(byte_data[1:5], byteorder='little', signed=True)
|
||||
|
||||
if cipher_type == 'rot13':
|
||||
return codecs.decode(ciphertext, cipher_type)
|
||||
try:
|
||||
byte_gen = _ByteGenerator(byte_data[0], seed)
|
||||
except ExtractorError as e:
|
||||
self.report_warning(f'Skipping format "{format_id}": {e.msg}')
|
||||
return None
|
||||
|
||||
self.report_warning(f'Skipping format "{format_id}": unsupported cipher type "{cipher_type}"')
|
||||
return None
|
||||
deciphered = bytearray(byte ^ next(byte_gen) for byte in byte_data[5:]).decode('latin-1')
|
||||
|
||||
return parsed_url._replace(path=f'/{deciphered}{path_remainder}').geturl()
|
||||
|
||||
def _fixup_formats(self, formats):
|
||||
for f in formats:
|
||||
@@ -333,8 +357,11 @@ class XHamsterIE(InfoExtractor):
|
||||
'height': get_height(quality),
|
||||
'filesize': format_sizes.get(quality),
|
||||
'http_headers': {
|
||||
'Referer': standard_url,
|
||||
'Referer': urlh.url,
|
||||
},
|
||||
# HTTP formats return "Wrong key" error even when deciphered by site JS
|
||||
# TODO: Remove this when resolved on the site's end
|
||||
'__needs_testing': True,
|
||||
})
|
||||
|
||||
categories_list = video.get('categories')
|
||||
@@ -371,7 +398,8 @@ class XHamsterIE(InfoExtractor):
|
||||
'age_limit': age_limit if age_limit is not None else 18,
|
||||
'categories': categories,
|
||||
'formats': self._fixup_formats(formats),
|
||||
'_format_sort_fields': ('res', 'proto', 'tbr'),
|
||||
# TODO: Revert to ('res', 'proto', 'tbr') when HTTP formats problem is resolved
|
||||
'_format_sort_fields': ('res', 'proto:m3u8', 'tbr'),
|
||||
}
|
||||
|
||||
# Old layout fallback
|
||||
|
||||
@@ -13,55 +13,16 @@ from ..utils import (
|
||||
parse_iso8601,
|
||||
traverse_obj,
|
||||
try_get,
|
||||
update_url,
|
||||
url_or_none,
|
||||
)
|
||||
|
||||
|
||||
class YahooIE(InfoExtractor):
|
||||
IE_DESC = 'Yahoo screen and movies'
|
||||
IE_NAME = 'yahoo'
|
||||
_VALID_URL = r'(?P<url>https?://(?:(?P<country>[a-zA-Z]{2}(?:-[a-zA-Z]{2})?|malaysia)\.)?(?:[\da-zA-Z_-]+\.)?yahoo\.com/(?:[^/]+/)*(?P<id>[^?&#]*-[0-9]+(?:-[a-z]+)?)\.html)'
|
||||
_EMBED_REGEX = [r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1']
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
|
||||
'info_dict': {
|
||||
'id': '2d25e626-2378-391f-ada0-ddaf1417e588',
|
||||
'ext': 'mp4',
|
||||
'title': 'Julian Smith & Travis Legg Watch Julian Smith',
|
||||
'description': 'Julian and Travis watch Julian Smith',
|
||||
'duration': 6863,
|
||||
'timestamp': 1369812016,
|
||||
'upload_date': '20130529',
|
||||
},
|
||||
'skip': 'No longer exists',
|
||||
}, {
|
||||
'url': 'https://screen.yahoo.com/community/community-sizzle-reel-203225340.html?format=embed',
|
||||
'md5': '7993e572fac98e044588d0b5260f4352',
|
||||
'info_dict': {
|
||||
'id': '4fe78544-8d48-39d8-97cd-13f205d9fcdb',
|
||||
'ext': 'mp4',
|
||||
'title': "Yahoo Saves 'Community'",
|
||||
'description': 'md5:4d4145af2fd3de00cbb6c1d664105053',
|
||||
'duration': 170,
|
||||
'timestamp': 1406838636,
|
||||
'upload_date': '20140731',
|
||||
},
|
||||
'skip': 'Unfortunately, this video is not available in your region',
|
||||
}, {
|
||||
'url': 'https://uk.screen.yahoo.com/editor-picks/cute-raccoon-freed-drain-using-091756545.html',
|
||||
'md5': '71298482f7c64cbb7fa064e4553ff1c1',
|
||||
'info_dict': {
|
||||
'id': 'b3affa53-2e14-3590-852b-0e0db6cd1a58',
|
||||
'ext': 'webm',
|
||||
'title': 'Cute Raccoon Freed From Drain\u00a0Using Angle Grinder',
|
||||
'description': 'md5:f66c890e1490f4910a9953c941dee944',
|
||||
'duration': 97,
|
||||
'timestamp': 1414489862,
|
||||
'upload_date': '20141028',
|
||||
},
|
||||
'skip': 'No longer exists',
|
||||
}, {
|
||||
'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
|
||||
'url': 'https://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
|
||||
'md5': '88e209b417f173d86186bef6e4d1f160',
|
||||
'info_dict': {
|
||||
'id': 'f885cf7f-43d4-3450-9fac-46ac30ece521',
|
||||
@@ -69,27 +30,33 @@ class YahooIE(InfoExtractor):
|
||||
'title': 'China Moses Is Crazy About the Blues',
|
||||
'description': 'md5:9900ab8cd5808175c7b3fe55b979bed0',
|
||||
'duration': 128,
|
||||
'timestamp': 1385722202,
|
||||
'timestamp': 1385721938,
|
||||
'upload_date': '20131129',
|
||||
'display_id': 'china-moses-crazy-blues-104538833',
|
||||
'view_count': int,
|
||||
'thumbnail': r're:https://media\.zenfs\.com/.+',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.yahoo.com/movies/v/true-story-trailer-173000497.html',
|
||||
'md5': '2a9752f74cb898af5d1083ea9f661b58',
|
||||
# 'md5': '989396ae73d20c6f057746fb226aa215', # varies between this and 'b17ac378b1134fa44370fb27db09a744'
|
||||
'info_dict': {
|
||||
'id': '071c4013-ce30-3a93-a5b2-e0413cd4a9d1',
|
||||
'ext': 'mp4',
|
||||
'title': '\'True Story\' Trailer',
|
||||
'description': 'True Story',
|
||||
'duration': 150,
|
||||
'timestamp': 1418919206,
|
||||
'timestamp': 1418923800,
|
||||
'upload_date': '20141218',
|
||||
'display_id': 'true-story-trailer-173000497',
|
||||
'view_count': int,
|
||||
'thumbnail': r're:https://media\.zenfs\.com/.+\.jpg',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://gma.yahoo.com/pizza-delivery-man-surprised-huge-tip-college-kids-195200785.html',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'note': 'NBC Sports embeds',
|
||||
'url': 'http://sports.yahoo.com/blogs/ncaab-the-dagger/tyler-kalinoski-s-buzzer-beater-caps-davidson-s-comeback-win-185609842.html?guid=nbc_cbk_davidsonbuzzerbeater_150313',
|
||||
'url': 'https://sports.yahoo.com/blogs/ncaab-the-dagger/tyler-kalinoski-s-buzzer-beater-caps-davidson-s-comeback-win-185609842.html?guid=nbc_cbk_davidsonbuzzerbeater_150313',
|
||||
'info_dict': {
|
||||
'id': '9CsDKds0kvHI',
|
||||
'ext': 'flv',
|
||||
@@ -99,26 +66,10 @@ class YahooIE(InfoExtractor):
|
||||
'uploader': 'NBCU-SPORTS',
|
||||
'timestamp': 1426270238,
|
||||
},
|
||||
'skip': 'Page no longer has video',
|
||||
}, {
|
||||
'url': 'https://tw.news.yahoo.com/-100120367.html',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
# Query result is embedded in webpage, but explicit request to video API fails with geo restriction
|
||||
'url': 'https://screen.yahoo.com/community/communitary-community-episode-1-ladders-154501237.html',
|
||||
'md5': '4fbafb9c9b6f07aa8f870629f6671b35',
|
||||
'info_dict': {
|
||||
'id': '1f32853c-a271-3eef-8cb6-f6d6872cb504',
|
||||
'ext': 'mp4',
|
||||
'title': 'Communitary - Community Episode 1: Ladders',
|
||||
'description': 'md5:8fc39608213295748e1e289807838c97',
|
||||
'duration': 1646,
|
||||
'timestamp': 1440436550,
|
||||
'upload_date': '20150824',
|
||||
'series': 'Communitary',
|
||||
'season_number': 6,
|
||||
'episode_number': 1,
|
||||
},
|
||||
'skip': 'No longer exists',
|
||||
}, {
|
||||
# ytwnews://cavideo/
|
||||
'url': 'https://tw.video.yahoo.com/movie-tw/單車天使-中文版預-092316541.html',
|
||||
@@ -129,12 +80,16 @@ class YahooIE(InfoExtractor):
|
||||
'description': '中文版預',
|
||||
'timestamp': 1476696196,
|
||||
'upload_date': '20161017',
|
||||
'view_count': int,
|
||||
'duration': 141,
|
||||
'thumbnail': r're:https://media\.zenfs\.com/.+\.jpg',
|
||||
'series': '電影',
|
||||
'display_id': '單車天使-中文版預-092316541',
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
},
|
||||
}, {
|
||||
# Contains both a Yahoo hosted video and multiple Youtube embeds
|
||||
'url': 'https://www.yahoo.com/entertainment/gwen-stefani-reveals-the-pop-hit-she-passed-on-assigns-it-to-her-voice-contestant-instead-033045672.html',
|
||||
'info_dict': {
|
||||
'id': '46c5d95a-528f-3d03-b732-732fcadd51de',
|
||||
@@ -147,24 +102,29 @@ class YahooIE(InfoExtractor):
|
||||
'ext': 'mp4',
|
||||
'title': 'Gwen Stefani reveals she turned down one of Sia\'s best songs',
|
||||
'description': 'On "The Voice" Tuesday, Gwen Stefani told Taylor Swift which Sia hit was almost hers.',
|
||||
'timestamp': 1572406500,
|
||||
'timestamp': 1572406499,
|
||||
'upload_date': '20191030',
|
||||
},
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '352CFDOQrKg',
|
||||
'ext': 'mp4',
|
||||
'title': 'Kyndal Inskeep "Performs the Hell Out of" Sia\'s "Elastic Heart" - The Voice Knockouts 2019',
|
||||
'description': 'md5:7fe8e3d5806f96002e55f190d1d94479',
|
||||
'uploader': 'The Voice',
|
||||
'uploader_id': 'NBCTheVoice',
|
||||
'upload_date': '20191029',
|
||||
'display_id': 'gwen-stefani-reveals-she-turned-033459311',
|
||||
'view_count': int,
|
||||
'duration': 97,
|
||||
'thumbnail': 'https://s.yimg.com/os/creatr-uploaded-images/2019-10/348bb330-fac6-11e9-8d27-38e85d573702',
|
||||
'series': 'Last Night Now',
|
||||
},
|
||||
}],
|
||||
'params': {
|
||||
'playlistend': 2,
|
||||
}, {
|
||||
'url': 'https://sports.yahoo.com/video/rams-lose-grip-nfcs-top-174614409.html',
|
||||
'info_dict': {
|
||||
'id': '6b15f100-cf5c-3ad0-9c96-87cbd2f72d4a',
|
||||
'ext': 'mp4',
|
||||
'display_id': 'rams-lose-grip-nfcs-top-174614409',
|
||||
'title': 'Rams lose their grip on NFC\'s top seed — can they still secure the bye?',
|
||||
'description': 'md5:5f4f98ab3c4de80e54c105b6bbb1d024',
|
||||
'view_count': int,
|
||||
'duration': 85,
|
||||
'thumbnail': 'https://s.yimg.com/os/creatr-uploaded-images/2025-12/94fc4840-dd02-11f0-beff-38ba3a4992e3',
|
||||
'timestamp': 1766166374,
|
||||
'upload_date': '20251219',
|
||||
},
|
||||
'expected_warnings': ['HTTP Error 404', 'Ignoring subtitle tracks'],
|
||||
}, {
|
||||
'url': 'https://malaysia.news.yahoo.com/video/bystanders-help-ontario-policeman-bust-190932818.html',
|
||||
'only_matching': True,
|
||||
@@ -178,14 +138,12 @@ class YahooIE(InfoExtractor):
|
||||
|
||||
def _extract_yahoo_video(self, video_id, country):
|
||||
video = self._download_json(
|
||||
f'https://{country}.yahoo.com/_td/api/resource/VideoService.videos;view=full;video_ids=["{video_id}"]',
|
||||
video_id, 'Downloading video JSON metadata')[0]
|
||||
title = video['title']
|
||||
|
||||
f'https://video-api.yql.yahoo.com/v1/video/sapi/streams/{video_id}',
|
||||
video_id, 'Downloading video JSON metadata')['query']['results']['mediaObj'][0]['meta']
|
||||
if country == 'malaysia':
|
||||
country = 'my'
|
||||
|
||||
is_live = video.get('live_state') == 'live'
|
||||
is_live = traverse_obj(video, ('uplynk_live', {bool})) is True
|
||||
fmts = ('m3u8',) if is_live else ('webm', 'mp4')
|
||||
|
||||
urls = []
|
||||
@@ -231,43 +189,23 @@ class YahooIE(InfoExtractor):
|
||||
'ext': mimetype2ext(cc.get('content_type')),
|
||||
})
|
||||
|
||||
streaming_url = video.get('streaming_url')
|
||||
if streaming_url and not is_live:
|
||||
formats.extend(self._extract_m3u8_formats(
|
||||
streaming_url, video_id, 'mp4',
|
||||
'm3u8_native', m3u8_id='hls', fatal=False))
|
||||
|
||||
if not formats and msg == 'geo restricted':
|
||||
self.raise_geo_restricted(metadata_available=True)
|
||||
|
||||
thumbnails = []
|
||||
for thumb in video.get('thumbnails', []):
|
||||
thumb_url = thumb.get('url')
|
||||
if not thumb_url:
|
||||
continue
|
||||
thumbnails.append({
|
||||
'id': thumb.get('tag'),
|
||||
'url': thumb.get('url'),
|
||||
'width': int_or_none(thumb.get('width')),
|
||||
'height': int_or_none(thumb.get('height')),
|
||||
})
|
||||
|
||||
series_info = video.get('series_info') or {}
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'formats': formats,
|
||||
'thumbnails': thumbnails,
|
||||
'description': clean_html(video.get('description')),
|
||||
'timestamp': parse_iso8601(video.get('publish_time')),
|
||||
'subtitles': subtitles,
|
||||
'duration': int_or_none(video.get('duration')),
|
||||
'view_count': int_or_none(video.get('view_count')),
|
||||
'is_live': is_live,
|
||||
'series': video.get('show_name'),
|
||||
'season_number': int_or_none(series_info.get('season_number')),
|
||||
'episode_number': int_or_none(series_info.get('episode_number')),
|
||||
**traverse_obj(video, {
|
||||
'title': ('title', {clean_html}),
|
||||
'description': ('description', {clean_html}),
|
||||
'thumbnail': ('thumbnail', {url_or_none}, {update_url(scheme='https')}),
|
||||
'timestamp': ('publish_time', {parse_iso8601}),
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'view_count': ('view_count', {int_or_none}),
|
||||
'series': ('show_name', {str}, filter),
|
||||
}),
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
@@ -321,14 +259,13 @@ class YahooIE(InfoExtractor):
|
||||
|
||||
|
||||
class YahooSearchIE(SearchInfoExtractor):
|
||||
IE_DESC = 'Yahoo screen search'
|
||||
_MAX_RESULTS = 1000
|
||||
IE_NAME = 'screen.yahoo:search'
|
||||
IE_NAME = 'yahoo:search'
|
||||
_SEARCH_KEY = 'yvsearch'
|
||||
|
||||
def _search_results(self, query):
|
||||
for pagenum in itertools.count(0):
|
||||
result_url = f'http://video.search.yahoo.com/search/?p={urllib.parse.quote_plus(query)}&fr=screen&o=js&gs=0&b={pagenum * 30}'
|
||||
result_url = f'https://video.search.yahoo.com/search/?p={urllib.parse.quote_plus(query)}&fr=screen&o=js&gs=0&b={pagenum * 30}'
|
||||
info = self._download_json(result_url, query,
|
||||
note='Downloading results page ' + str(pagenum + 1))
|
||||
yield from (self.url_result(result['rurl']) for result in info['results'])
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
int_or_none,
|
||||
join_nonempty,
|
||||
remove_end,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class YfanefaIE(InfoExtractor):
|
||||
IE_NAME = 'yfanefa'
|
||||
_VALID_URL = r'https?://(?:www\.)?yfanefa\.com/(?P<id>[^?#]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.yfanefa.com/record/2717',
|
||||
'info_dict': {
|
||||
'id': 'record-2717',
|
||||
'ext': 'mp4',
|
||||
'title': 'THE HALLAMSHIRE RIFLES LEAVING SHEFFIELD, 1914',
|
||||
'duration': 5239,
|
||||
'thumbnail': r're:https://media\.yfanefa\.com/storage/v1/file/',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.yfanefa.com/news/53',
|
||||
'info_dict': {
|
||||
'id': 'news-53',
|
||||
'ext': 'mp4',
|
||||
'title': 'Memory Bank: Bradford Launch',
|
||||
'thumbnail': r're:https://media\.yfanefa\.com/storage/v1/file/',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.yfanefa.com/evaluating_nature_matters',
|
||||
'info_dict': {
|
||||
'id': 'evaluating_nature_matters',
|
||||
'ext': 'mp4',
|
||||
'title': 'Evaluating Nature Matters',
|
||||
'thumbnail': r're:https://media\.yfanefa\.com/storage/v1/file/',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
player_data = self._search_json(
|
||||
r'iwPlayer\.options\["[\w.]+"\]\s*=', webpage, 'player options', video_id)
|
||||
|
||||
formats = []
|
||||
video_url = join_nonempty(player_data['url'], player_data.get('signature'), delim='')
|
||||
if determine_ext(video_url) == 'm3u8':
|
||||
formats = self._extract_m3u8_formats(
|
||||
video_url, video_id, 'mp4', m3u8_id='hls')
|
||||
else:
|
||||
formats = [{'url': video_url, 'ext': 'mp4'}]
|
||||
|
||||
return {
|
||||
'id': video_id.strip('/').replace('/', '-'),
|
||||
'title':
|
||||
self._og_search_title(webpage, default=None)
|
||||
or remove_end(self._html_extract_title(webpage), ' | Yorkshire Film Archive'),
|
||||
'formats': formats,
|
||||
**traverse_obj(player_data, {
|
||||
'thumbnail': ('preview', {url_or_none}),
|
||||
'duration': ('duration', {int_or_none}),
|
||||
}),
|
||||
}
|
||||
@@ -104,6 +104,7 @@ INNERTUBE_CLIENTS = {
|
||||
},
|
||||
'INNERTUBE_CONTEXT_CLIENT_NAME': 1,
|
||||
'SUPPORTS_COOKIES': True,
|
||||
'SUPPORTS_AD_PLAYBACK_CONTEXT': True,
|
||||
**WEB_PO_TOKEN_POLICIES,
|
||||
},
|
||||
# Safari UA returns pre-merged video+audio 144p/240p/360p/720p/1080p HLS formats
|
||||
@@ -117,6 +118,7 @@ INNERTUBE_CLIENTS = {
|
||||
},
|
||||
'INNERTUBE_CONTEXT_CLIENT_NAME': 1,
|
||||
'SUPPORTS_COOKIES': True,
|
||||
'SUPPORTS_AD_PLAYBACK_CONTEXT': True,
|
||||
**WEB_PO_TOKEN_POLICIES,
|
||||
},
|
||||
'web_embedded': {
|
||||
@@ -157,6 +159,7 @@ INNERTUBE_CLIENTS = {
|
||||
),
|
||||
},
|
||||
'SUPPORTS_COOKIES': True,
|
||||
'SUPPORTS_AD_PLAYBACK_CONTEXT': True,
|
||||
},
|
||||
# This client now requires sign-in for every video
|
||||
'web_creator': {
|
||||
@@ -313,6 +316,7 @@ INNERTUBE_CLIENTS = {
|
||||
),
|
||||
},
|
||||
'SUPPORTS_COOKIES': True,
|
||||
'SUPPORTS_AD_PLAYBACK_CONTEXT': True,
|
||||
},
|
||||
'tv': {
|
||||
'INNERTUBE_CONTEXT': {
|
||||
@@ -327,6 +331,17 @@ INNERTUBE_CLIENTS = {
|
||||
# See: https://github.com/youtube/cobalt/blob/main/cobalt/browser/user_agent/user_agent_platform_info.cc#L506
|
||||
'AUTHENTICATED_USER_AGENT': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/25.lts.30.1034943-gold (unlike Gecko), Unknown_TV_Unknown_0/Unknown (Unknown, Unknown)',
|
||||
},
|
||||
'tv_downgraded': {
|
||||
'INNERTUBE_CONTEXT': {
|
||||
'client': {
|
||||
'clientName': 'TVHTML5',
|
||||
'clientVersion': '5.20251105',
|
||||
'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version',
|
||||
},
|
||||
},
|
||||
'INNERTUBE_CONTEXT_CLIENT_NAME': 7,
|
||||
'SUPPORTS_COOKIES': True,
|
||||
},
|
||||
'tv_simply': {
|
||||
'INNERTUBE_CONTEXT': {
|
||||
'client': {
|
||||
@@ -380,11 +395,15 @@ def short_client_name(client_name):
|
||||
return join_nonempty(main[:4], ''.join(x[0] for x in parts)).upper()
|
||||
|
||||
|
||||
def build_innertube_clients():
|
||||
THIRD_PARTY = {
|
||||
def _fix_embedded_ytcfg(ytcfg):
|
||||
ytcfg['INNERTUBE_CONTEXT'].setdefault('thirdParty', {}).update({
|
||||
'embedUrl': 'https://www.youtube.com/', # Can be any valid URL
|
||||
}
|
||||
BASE_CLIENTS = ('ios', 'web', 'tv', 'mweb', 'android')
|
||||
})
|
||||
|
||||
|
||||
def build_innertube_clients():
|
||||
# From highest to lowest priority
|
||||
BASE_CLIENTS = ('tv', 'web', 'mweb', 'android', 'ios')
|
||||
priority = qualities(BASE_CLIENTS[::-1])
|
||||
|
||||
for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
|
||||
@@ -397,6 +416,7 @@ def build_innertube_clients():
|
||||
ytcfg.setdefault('SUBS_PO_TOKEN_POLICY', SubsPoTokenPolicy())
|
||||
ytcfg.setdefault('REQUIRE_AUTH', False)
|
||||
ytcfg.setdefault('SUPPORTS_COOKIES', False)
|
||||
ytcfg.setdefault('SUPPORTS_AD_PLAYBACK_CONTEXT', False)
|
||||
ytcfg.setdefault('PLAYER_PARAMS', None)
|
||||
ytcfg.setdefault('AUTHENTICATED_USER_AGENT', None)
|
||||
ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
|
||||
@@ -405,10 +425,7 @@ def build_innertube_clients():
|
||||
ytcfg['priority'] = 10 * priority(base_client)
|
||||
|
||||
if variant == 'embedded':
|
||||
ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
|
||||
ytcfg['priority'] -= 2
|
||||
elif variant:
|
||||
ytcfg['priority'] -= 3
|
||||
_fix_embedded_ytcfg(ytcfg)
|
||||
|
||||
|
||||
build_innertube_clients()
|
||||
@@ -991,6 +1008,10 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
|
||||
|
||||
ytcfg = self.extract_ytcfg(video_id, webpage) or {}
|
||||
|
||||
# See https://github.com/yt-dlp/yt-dlp/issues/14826
|
||||
if _split_innertube_client(client)[2] == 'embedded':
|
||||
_fix_embedded_ytcfg(ytcfg)
|
||||
|
||||
# Workaround for https://github.com/yt-dlp/yt-dlp/issues/12563
|
||||
# But it's not effective when logged-in
|
||||
if client == 'tv' and not self.is_authenticated:
|
||||
@@ -1044,7 +1065,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
|
||||
return next_continuation
|
||||
|
||||
return traverse_obj(renderer, (
|
||||
('contents', 'items', 'rows'), ..., 'continuationItemRenderer',
|
||||
('contents', 'items', 'rows', 'subThreads'), ..., 'continuationItemRenderer',
|
||||
('continuationEndpoint', ('button', 'buttonRenderer', 'command')),
|
||||
), get_all=False, expected_type=cls._extract_continuation_ep_data)
|
||||
|
||||
|
||||
@@ -340,8 +340,9 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
|
||||
thumbnails=self._extract_thumbnails(view_model, (
|
||||
'contentImage', *thumb_keys, 'thumbnailViewModel', 'image'), final_key='sources'),
|
||||
duration=traverse_obj(view_model, (
|
||||
'contentImage', 'thumbnailViewModel', 'overlays', ..., 'thumbnailOverlayBadgeViewModel',
|
||||
'thumbnailBadges', ..., 'thumbnailBadgeViewModel', 'text', {parse_duration}, any)),
|
||||
'contentImage', 'thumbnailViewModel', 'overlays', ...,
|
||||
(('thumbnailBottomOverlayViewModel', 'badges'), ('thumbnailOverlayBadgeViewModel', 'thumbnailBadges')),
|
||||
..., 'thumbnailBadgeViewModel', 'text', {parse_duration}, any)),
|
||||
timestamp=(traverse_obj(view_model, (
|
||||
'metadata', 'lockupMetadataViewModel', 'metadata', 'contentMetadataViewModel', 'metadataRows',
|
||||
..., 'metadataParts', ..., 'text', 'content', {lambda t: self._parse_time_text(t, report_failure=False)}, any))
|
||||
@@ -381,7 +382,8 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
|
||||
('accessibilityText', {lambda x: re.fullmatch(r'(.+), (?:[\d,.]+(?:[KM]| million)?|No) views? - play Short', x)}, 1)), any),
|
||||
'view_count': ('overlayMetadata', 'secondaryText', 'content', {parse_count}),
|
||||
}),
|
||||
thumbnails=self._extract_thumbnails(renderer, 'thumbnail', final_key='sources'))
|
||||
thumbnails=self._extract_thumbnails(
|
||||
renderer, ('thumbnailViewModel', 'thumbnailViewModel', 'image'), final_key='sources'))
|
||||
return
|
||||
|
||||
def _video_entry(self, video_renderer):
|
||||
@@ -1584,7 +1586,6 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
|
||||
'playlist_count': 50,
|
||||
'expected_warnings': ['YouTube Music is not directly supported'],
|
||||
}, {
|
||||
# TODO: fix test suite, 208163447408c78673b08c172beafe5c310fb167 broke this test
|
||||
'note': 'unlisted single video playlist',
|
||||
'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQLfIN0MMgp0wVV6MP3bM4_',
|
||||
'info_dict': {
|
||||
@@ -1884,8 +1885,6 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
|
||||
'playlist_mincount': 30,
|
||||
}, {
|
||||
# Shorts url result in shorts tab
|
||||
# TODO: Fix channel id extraction
|
||||
# TODO: fix test suite, 208163447408c78673b08c172beafe5c310fb167 broke this test
|
||||
'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/shorts',
|
||||
'info_dict': {
|
||||
'id': 'UCiu-3thuViMebBjw_5nWYrA',
|
||||
@@ -1914,7 +1913,6 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
|
||||
'params': {'extract_flat': True},
|
||||
}, {
|
||||
# Live video status should be extracted
|
||||
# TODO: fix test suite, 208163447408c78673b08c172beafe5c310fb167 broke this test
|
||||
'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/live',
|
||||
'info_dict': {
|
||||
'id': 'UCQvWX73GQygcwXOTSf_VDVg',
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,132 @@
|
||||
# YoutubeIE JS Challenge Provider Framework
|
||||
|
||||
As part of the YouTube extractor, we have a framework for solving n/sig JS Challenges programmatically. This can be used by plugins.
|
||||
|
||||
> [!TIP]
|
||||
> If publishing a JS Challenge Provider plugin to GitHub, add the [yt-dlp-jsc-provider](https://github.com/topics/yt-dlp-jsc-provider) topic to your repository to help users find it.
|
||||
|
||||
|
||||
## Public APIs
|
||||
|
||||
- `yt_dlp.extractor.youtube.jsc.provider`
|
||||
|
||||
Everything else is **internal-only** and no guarantees are made about the API stability.
|
||||
|
||||
> [!WARNING]
|
||||
> We will try our best to maintain stability with the public APIs.
|
||||
> However, due to the nature of extractors and YouTube, we may need to remove or change APIs in the future.
|
||||
> If you are using these APIs outside yt-dlp plugins, please account for this by importing them safely.
|
||||
|
||||
## JS Challenge Provider
|
||||
|
||||
`yt_dlp.extractor.youtube.jsc.provider`
|
||||
|
||||
```python
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
register_provider,
|
||||
register_preference,
|
||||
JsChallengeProvider,
|
||||
JsChallengeRequest,
|
||||
JsChallengeResponse,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeProviderRejectedRequest,
|
||||
JsChallengeType,
|
||||
JsChallengeProviderResponse,
|
||||
NChallengeOutput,
|
||||
)
|
||||
from yt_dlp.utils import traverse_obj, Popen
|
||||
import json
|
||||
import subprocess
|
||||
import typing
|
||||
|
||||
@register_provider
|
||||
class MyJsChallengeProviderJCP(JsChallengeProvider): # Provider class name must end with "JCP"
|
||||
PROVIDER_VERSION = '0.2.1'
|
||||
# Define a unique display name for the provider
|
||||
PROVIDER_NAME = 'my-provider'
|
||||
BUG_REPORT_LOCATION = 'https://issues.example.com/report'
|
||||
|
||||
# Set supported challenge types.
|
||||
# If None, the provider will handle all types.
|
||||
_SUPPORTED_TYPES = [JsChallengeType.N]
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""
|
||||
Check if the provider is available (e.g. all required dependencies are available)
|
||||
This is used to determine if the provider should be used and to provide debug information.
|
||||
|
||||
IMPORTANT: This method SHOULD NOT make any network requests or perform any expensive operations.
|
||||
|
||||
Since this is called multiple times, we recommend caching the result.
|
||||
"""
|
||||
return True
|
||||
|
||||
def close(self):
|
||||
# Optional close hook, called when YoutubeDL is closed.
|
||||
pass
|
||||
|
||||
def _real_bulk_solve(self, requests: list[JsChallengeRequest]) -> typing.Generator[JsChallengeProviderResponse, None, None]:
|
||||
# ℹ️ If you need to do additional validation on the requests.
|
||||
# Raise yt_dlp.extractor.youtube.jsc.provider.JsChallengeProviderRejectedRequest if the request is not supported.
|
||||
if len("something") > 255:
|
||||
raise JsChallengeProviderRejectedRequest('Challenges longer than 255 are not supported', expected=True)
|
||||
|
||||
|
||||
# ℹ️ Settings are pulled from extractor args passed to yt-dlp with the key `youtubejsc-<PROVIDER_KEY>`.
|
||||
# For this example, the extractor arg would be:
|
||||
# `--extractor-args "youtubejsc-myjschallengeprovider:bin_path=/path/to/bin"`
|
||||
bin_path = self._configuration_arg(
|
||||
'bin_path', default=['/path/to/bin'])[0]
|
||||
|
||||
# See below for logging guidelines
|
||||
self.logger.trace(f'Using bin path: {bin_path}')
|
||||
|
||||
for request in requests:
|
||||
# You can use the _get_player method to get the player JS code if needed.
|
||||
# This shares the same caching as the YouTube extractor, so it will not make unnecessary requests.
|
||||
player_js = self._get_player(request.video_id, request.input.player_url)
|
||||
cmd = f'{bin_path} {request.input.challenges} {player_js}'
|
||||
self.logger.info(f'Executing command: {cmd}')
|
||||
stdout, _, ret = Popen.run(cmd, text=True, shell=True, stdout=subprocess.PIPE)
|
||||
if ret != 0:
|
||||
# ℹ️ If there is an error, raise JsChallengeProviderError.
|
||||
# The request will be sent to the next provider if there is one.
|
||||
# You can specify whether it is expected or not. If it is unexpected,
|
||||
# the log will include a link to the bug report location (BUG_REPORT_LOCATION).
|
||||
|
||||
# raise JsChallengeProviderError(f'Command returned error code {ret}', expected=False)
|
||||
|
||||
# You can also only fail this specific request by returning a JsChallengeProviderResponse with the error.
|
||||
# This will allow other requests to be processed by this provider.
|
||||
yield JsChallengeProviderResponse(
|
||||
request=request,
|
||||
error=JsChallengeProviderError(f'Command returned error code {ret}', expected=False)
|
||||
)
|
||||
|
||||
yield JsChallengeProviderResponse(
|
||||
request=request,
|
||||
response=JsChallengeResponse(
|
||||
type=JsChallengeType.N,
|
||||
output=NChallengeOutput(results=traverse_obj(json.loads(stdout))),
|
||||
))
|
||||
|
||||
|
||||
# If there are multiple JS Challenge Providers that can handle the same JsChallengeRequest(s),
|
||||
# you can define a preference function to increase/decrease the priority of providers.
|
||||
|
||||
@register_preference(MyJsChallengeProviderJCP)
|
||||
def my_provider_preference(provider: JsChallengeProvider, requests: list[JsChallengeRequest]) -> int:
|
||||
return 50
|
||||
```
|
||||
|
||||
## Logging Guidelines
|
||||
|
||||
- Use the `self.logger` object to log messages.
|
||||
- When making HTTP requests or any other time-expensive operation, use `self.logger.info` to log a message to standard non-verbose output.
|
||||
- This lets users know what is happening when a time-expensive operation is taking place.
|
||||
- Technical information such as a command being executed should be logged to `self.logger.debug`
|
||||
- Use `self.logger.trace` for very detailed information that is only useful for debugging to avoid cluttering the debug log.
|
||||
|
||||
## Debugging
|
||||
|
||||
- Use `-v --extractor-args "youtube:jsc_trace=true"` to enable JS Challenge debug output.
|
||||
@@ -0,0 +1,5 @@
|
||||
# Trigger import of built-in providers
|
||||
from ._builtin.bun import BunJCP as _BunJCP # noqa: F401
|
||||
from ._builtin.deno import DenoJCP as _DenoJCP # noqa: F401
|
||||
from ._builtin.node import NodeJCP as _NodeJCP # noqa: F401
|
||||
from ._builtin.quickjs import QuickJSJCP as _QuickJSJCP # noqa: F401
|
||||
@@ -0,0 +1,150 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import urllib.parse
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import (
|
||||
_EJS_WIKI_URL,
|
||||
EJSBaseJCP,
|
||||
Script,
|
||||
ScriptSource,
|
||||
ScriptType,
|
||||
ScriptVariant,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.vendor import load_script
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeRequest,
|
||||
register_preference,
|
||||
register_provider,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider
|
||||
from yt_dlp.extractor.youtube.pot.provider import provider_bug_report_message
|
||||
from yt_dlp.utils import Popen
|
||||
from yt_dlp.utils.networking import HTTPHeaderDict, clean_proxies
|
||||
|
||||
# KNOWN ISSUES:
|
||||
# - If node_modules is present and includes a requested lib, the version we request is ignored
|
||||
# and whatever installed in node_modules is used.
|
||||
# - No way to ignore existing node_modules, lock files, etc.
|
||||
# - No sandboxing options available
|
||||
# - Cannot detect if npm packages are cached without potentially downloading them.
|
||||
# `--no-install` appears to disable the cache.
|
||||
# - npm auto-install may fail with an integrity error when using HTTP proxies
|
||||
# - npm auto-install HTTP proxy support may be limited on older Bun versions
|
||||
# - Cannot disable the transpiler / specify lang for stdin
|
||||
|
||||
|
||||
@register_provider
|
||||
class BunJCP(EJSBaseJCP, BuiltinIEContentProvider):
|
||||
PROVIDER_NAME = 'bun'
|
||||
JS_RUNTIME_NAME = 'bun'
|
||||
BUN_NPM_LIB_FILENAME = 'yt.solver.bun.lib.js'
|
||||
SUPPORTED_PROXY_SCHEMES = ['http', 'https']
|
||||
|
||||
def _iter_script_sources(self):
|
||||
yield from super()._iter_script_sources()
|
||||
yield ScriptSource.BUILTIN, self._bun_npm_source
|
||||
|
||||
def _bun_npm_source(self, script_type: ScriptType, /):
|
||||
if script_type != ScriptType.LIB:
|
||||
return None
|
||||
if 'ejs:npm' not in self.ie.get_param('remote_components', []):
|
||||
return self._skip_component('ejs:npm')
|
||||
|
||||
# Check to see if the environment proxies are compatible with Bun npm source
|
||||
if unsupported_scheme := self._check_env_proxies(self._get_env_options()):
|
||||
self.logger.warning(
|
||||
f'Bun NPM package downloads only support HTTP/HTTPS proxies; skipping remote NPM package downloads. '
|
||||
f'Provide another distribution of the challenge solver script or use '
|
||||
f'another JS runtime that supports "{unsupported_scheme}" proxies. '
|
||||
f'For more information and alternatives, refer to {_EJS_WIKI_URL}')
|
||||
return None
|
||||
|
||||
# Bun-specific lib scripts that uses Bun autoimport
|
||||
# https://bun.com/docs/runtime/autoimport
|
||||
error_hook = lambda e: self.logger.warning(
|
||||
f'Failed to read bun challenge solver lib script: {e}{provider_bug_report_message(self)}')
|
||||
code = load_script(
|
||||
self.BUN_NPM_LIB_FILENAME, error_hook=error_hook)
|
||||
if code:
|
||||
return Script(script_type, ScriptVariant.BUN_NPM, ScriptSource.BUILTIN, self._SCRIPT_VERSION, code)
|
||||
return None
|
||||
|
||||
def _check_env_proxies(self, env):
|
||||
# check that the schemes of both HTTP_PROXY and HTTPS_PROXY are supported
|
||||
for key in ('HTTP_PROXY', 'HTTPS_PROXY'):
|
||||
proxy = env.get(key)
|
||||
if not proxy:
|
||||
continue
|
||||
scheme = urllib.parse.urlparse(proxy).scheme.lower()
|
||||
if scheme not in self.SUPPORTED_PROXY_SCHEMES:
|
||||
return scheme
|
||||
return None
|
||||
|
||||
def _get_env_options(self) -> dict[str, str]:
|
||||
options = os.environ.copy() # pass through existing bun env vars
|
||||
request_proxies = self.ie._downloader.proxies.copy()
|
||||
clean_proxies(request_proxies, HTTPHeaderDict())
|
||||
|
||||
# Apply 'all' proxy first, then allow per-scheme overrides
|
||||
if request_proxies.get('all') is not None:
|
||||
options['HTTP_PROXY'] = options['HTTPS_PROXY'] = request_proxies['all']
|
||||
for key, env in (('http', 'HTTP_PROXY'), ('https', 'HTTPS_PROXY')):
|
||||
val = request_proxies.get(key)
|
||||
if val is not None:
|
||||
options[env] = val
|
||||
if self.ie.get_param('nocheckcertificate'):
|
||||
options['NODE_TLS_REJECT_UNAUTHORIZED'] = '0'
|
||||
|
||||
# Disable Bun transpiler cache
|
||||
options['BUN_RUNTIME_TRANSPILER_CACHE_PATH'] = '0'
|
||||
|
||||
# Prevent segfault: <https://github.com/oven-sh/bun/issues/22901>
|
||||
options.pop('JSC_useJIT', None)
|
||||
if self.ejs_setting('jitless', ['false']) != ['false']:
|
||||
options['BUN_JSC_useJIT'] = '0'
|
||||
|
||||
return options
|
||||
|
||||
def _run_js_runtime(self, stdin: str, /) -> str:
|
||||
# https://bun.com/docs/cli/run
|
||||
options = ['--no-addons', '--prefer-offline']
|
||||
if self._lib_script.variant == ScriptVariant.BUN_NPM:
|
||||
# Enable auto-install even if node_modules is present
|
||||
options.append('--install=fallback')
|
||||
else:
|
||||
options.append('--no-install')
|
||||
cmd = [self.runtime_info.path, '--bun', 'run', *options, '-']
|
||||
self.logger.debug(f'Running bun: {shlex.join(cmd)}')
|
||||
|
||||
with Popen(
|
||||
cmd,
|
||||
text=True,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=self._get_env_options(),
|
||||
) as proc:
|
||||
stdout, stderr = proc.communicate_or_kill(stdin)
|
||||
stderr = self._clean_stderr(stderr)
|
||||
if proc.returncode or stderr:
|
||||
msg = f'Error running bun process (returncode: {proc.returncode})'
|
||||
if stderr:
|
||||
msg = f'{msg}: {stderr.strip()}'
|
||||
raise JsChallengeProviderError(msg)
|
||||
return stdout
|
||||
|
||||
def _clean_stderr(self, stderr):
|
||||
return '\n'.join(
|
||||
line for line in stderr.splitlines()
|
||||
if not re.match(r'^Bun v\d+\.\d+\.\d+ \([\w\s]+\)$', line))
|
||||
|
||||
|
||||
@register_preference(BunJCP)
|
||||
def preference(provider: JsChallengeProvider, requests: list[JsChallengeRequest]) -> int:
|
||||
return 800
|
||||
@@ -0,0 +1,131 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import (
|
||||
EJSBaseJCP,
|
||||
Script,
|
||||
ScriptSource,
|
||||
ScriptType,
|
||||
ScriptVariant,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.vendor import load_script
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeRequest,
|
||||
register_preference,
|
||||
register_provider,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider
|
||||
from yt_dlp.extractor.youtube.pot.provider import provider_bug_report_message
|
||||
from yt_dlp.utils import Popen, remove_terminal_sequences
|
||||
from yt_dlp.utils.networking import HTTPHeaderDict, clean_proxies
|
||||
|
||||
# KNOWN ISSUES:
|
||||
# - Can't avoid analysis cache: https://github.com/yt-dlp/yt-dlp/pull/14849#issuecomment-3475840821
|
||||
|
||||
|
||||
@register_provider
|
||||
class DenoJCP(EJSBaseJCP, BuiltinIEContentProvider):
|
||||
PROVIDER_NAME = 'deno'
|
||||
JS_RUNTIME_NAME = 'deno'
|
||||
|
||||
_DENO_BASE_OPTIONS = [
|
||||
'--ext=js', '--no-code-cache', '--no-prompt', '--no-remote',
|
||||
'--no-lock', '--node-modules-dir=none', '--no-config',
|
||||
]
|
||||
DENO_NPM_LIB_FILENAME = 'yt.solver.deno.lib.js'
|
||||
_NPM_PACKAGES_CACHED = False
|
||||
|
||||
def _iter_script_sources(self):
|
||||
yield from super()._iter_script_sources()
|
||||
yield ScriptSource.BUILTIN, self._deno_npm_source
|
||||
|
||||
def _deno_npm_source(self, script_type: ScriptType, /):
|
||||
if script_type != ScriptType.LIB:
|
||||
return None
|
||||
# Deno-specific lib scripts that use Deno NPM imports
|
||||
error_hook = lambda e: self.logger.warning(
|
||||
f'Failed to read deno challenge solver lib script: {e}{provider_bug_report_message(self)}')
|
||||
code = load_script(
|
||||
self.DENO_NPM_LIB_FILENAME, error_hook=error_hook)
|
||||
if not code:
|
||||
return None
|
||||
if 'ejs:npm' not in self.ie.get_param('remote_components', []):
|
||||
# We may still be able to continue if the npm packages are available/cached
|
||||
self._NPM_PACKAGES_CACHED = self._npm_packages_cached(code)
|
||||
if not self._NPM_PACKAGES_CACHED:
|
||||
return self._skip_component('ejs:npm')
|
||||
return Script(script_type, ScriptVariant.DENO_NPM, ScriptSource.BUILTIN, self._SCRIPT_VERSION, code)
|
||||
|
||||
def _npm_packages_cached(self, stdin: str) -> bool:
|
||||
# Check if npm packages are cached, so we can run without --remote-components ejs:npm
|
||||
self.logger.debug('Checking if npm packages are cached')
|
||||
try:
|
||||
self._run_deno(stdin, [*self._DENO_BASE_OPTIONS, '--cached-only'])
|
||||
except JsChallengeProviderError as e:
|
||||
self.logger.trace(f'Deno npm packages not cached: {e}')
|
||||
return False
|
||||
return True
|
||||
|
||||
def _run_js_runtime(self, stdin: str, /) -> str:
|
||||
options = [*self._DENO_BASE_OPTIONS]
|
||||
if self._lib_script.variant == ScriptVariant.DENO_NPM and self._NPM_PACKAGES_CACHED:
|
||||
options.append('--cached-only')
|
||||
elif self._lib_script.variant != ScriptVariant.DENO_NPM:
|
||||
options.append('--no-npm')
|
||||
options.append('--cached-only')
|
||||
if self.ie.get_param('nocheckcertificate'):
|
||||
options.append('--unsafely-ignore-certificate-errors')
|
||||
# XXX: Convert this extractor-arg into a general option if/when a JSI framework is implemented
|
||||
if self.ejs_setting('jitless', ['false']) != ['false']:
|
||||
options.append('--v8-flags=--jitless')
|
||||
return self._run_deno(stdin, options)
|
||||
|
||||
def _get_env_options(self) -> dict[str, str]:
|
||||
options = os.environ.copy() # pass through existing deno env vars
|
||||
request_proxies = self.ie._downloader.proxies.copy()
|
||||
clean_proxies(request_proxies, HTTPHeaderDict())
|
||||
# Apply 'all' proxy first, then allow per-scheme overrides
|
||||
if 'all' in request_proxies and request_proxies['all'] is not None:
|
||||
options['HTTP_PROXY'] = options['HTTPS_PROXY'] = request_proxies['all']
|
||||
for key, env in (('http', 'HTTP_PROXY'), ('https', 'HTTPS_PROXY'), ('no', 'NO_PROXY')):
|
||||
if key in request_proxies and request_proxies[key] is not None:
|
||||
options[env] = request_proxies[key]
|
||||
return options
|
||||
|
||||
def _run_deno(self, stdin, options) -> str:
|
||||
cmd = [self.runtime_info.path, 'run', *options, '-']
|
||||
self.logger.debug(f'Running deno: {shlex.join(cmd)}')
|
||||
with Popen(
|
||||
cmd,
|
||||
text=True,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=self._get_env_options(),
|
||||
) as proc:
|
||||
stdout, stderr = proc.communicate_or_kill(stdin)
|
||||
stderr = self._clean_stderr(stderr)
|
||||
if proc.returncode or stderr:
|
||||
msg = f'Error running deno process (returncode: {proc.returncode})'
|
||||
if stderr:
|
||||
msg = f'{msg}: {stderr.strip()}'
|
||||
raise JsChallengeProviderError(msg)
|
||||
return stdout
|
||||
|
||||
def _clean_stderr(self, stderr):
|
||||
return '\n'.join(
|
||||
line for line in stderr.splitlines()
|
||||
if not (
|
||||
re.match(r'^Download\s+https\S+$', remove_terminal_sequences(line))
|
||||
or re.match(r'DANGER: TLS certificate validation is disabled for all hostnames', remove_terminal_sequences(line))))
|
||||
|
||||
|
||||
@register_preference(DenoJCP)
|
||||
def preference(provider: JsChallengeProvider, requests: list[JsChallengeRequest]) -> int:
|
||||
return 1000
|
||||
@@ -0,0 +1,328 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import dataclasses
|
||||
import enum
|
||||
import functools
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
from yt_dlp.dependencies import yt_dlp_ejs as _has_ejs
|
||||
from yt_dlp.extractor.youtube.jsc._builtin import vendor
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeProviderRejectedRequest,
|
||||
JsChallengeProviderResponse,
|
||||
JsChallengeResponse,
|
||||
JsChallengeType,
|
||||
NChallengeOutput,
|
||||
SigChallengeOutput,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import configuration_arg
|
||||
from yt_dlp.extractor.youtube.pot.provider import provider_bug_report_message
|
||||
from yt_dlp.utils import version_tuple
|
||||
from yt_dlp.utils._jsruntime import JsRuntimeInfo
|
||||
|
||||
if _has_ejs:
|
||||
import yt_dlp_ejs.yt.solver
|
||||
|
||||
TYPE_CHECKING = False
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Generator
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc.provider import JsChallengeRequest
|
||||
|
||||
_EJS_WIKI_URL = 'https://github.com/yt-dlp/yt-dlp/wiki/EJS'
|
||||
|
||||
|
||||
class ScriptType(enum.Enum):
|
||||
LIB = 'lib'
|
||||
CORE = 'core'
|
||||
|
||||
|
||||
class ScriptVariant(enum.Enum):
|
||||
UNKNOWN = 'unknown'
|
||||
MINIFIED = 'minified'
|
||||
UNMINIFIED = 'unminified'
|
||||
DENO_NPM = 'deno_npm'
|
||||
BUN_NPM = 'bun_npm'
|
||||
|
||||
|
||||
class ScriptSource(enum.Enum):
|
||||
PYPACKAGE = 'python package' # PyPI, PyInstaller exe, zipimport binary, etc
|
||||
CACHE = 'cache' # GitHub release assets (cached)
|
||||
WEB = 'web' # GitHub release assets (downloaded)
|
||||
BUILTIN = 'builtin' # vendored (full core script; import-only lib script + NPM cache)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Script:
|
||||
type: ScriptType
|
||||
variant: ScriptVariant
|
||||
source: ScriptSource
|
||||
version: str
|
||||
code: str
|
||||
|
||||
@functools.cached_property
|
||||
def hash(self, /) -> str:
|
||||
return hashlib.sha3_512(self.code.encode()).hexdigest()
|
||||
|
||||
def __str__(self, /):
|
||||
return f'<Script {self.type.value!r} v{self.version} (source: {self.source.value}) variant={self.variant.value!r} size={len(self.code)} hash={self.hash[:7]}...>'
|
||||
|
||||
|
||||
class EJSBaseJCP(JsChallengeProvider):
|
||||
JS_RUNTIME_NAME: str
|
||||
_CACHE_SECTION = 'challenge-solver'
|
||||
|
||||
_REPOSITORY = 'yt-dlp/ejs'
|
||||
_SUPPORTED_TYPES = [JsChallengeType.N, JsChallengeType.SIG]
|
||||
_SCRIPT_VERSION = vendor.VERSION
|
||||
# TODO: Integration tests for each kind of scripts source
|
||||
_ALLOWED_HASHES = {
|
||||
ScriptType.LIB: {
|
||||
ScriptVariant.UNMINIFIED: vendor.HASHES['yt.solver.lib.js'],
|
||||
ScriptVariant.MINIFIED: vendor.HASHES['yt.solver.lib.min.js'],
|
||||
ScriptVariant.DENO_NPM: vendor.HASHES['yt.solver.deno.lib.js'],
|
||||
ScriptVariant.BUN_NPM: vendor.HASHES['yt.solver.bun.lib.js'],
|
||||
},
|
||||
ScriptType.CORE: {
|
||||
ScriptVariant.MINIFIED: vendor.HASHES['yt.solver.core.min.js'],
|
||||
ScriptVariant.UNMINIFIED: vendor.HASHES['yt.solver.core.js'],
|
||||
},
|
||||
}
|
||||
|
||||
_SCRIPT_FILENAMES = {
|
||||
ScriptType.LIB: 'yt.solver.lib.js',
|
||||
ScriptType.CORE: 'yt.solver.core.js',
|
||||
}
|
||||
|
||||
_MIN_SCRIPT_FILENAMES = {
|
||||
ScriptType.LIB: 'yt.solver.lib.min.js',
|
||||
ScriptType.CORE: 'yt.solver.core.min.js',
|
||||
}
|
||||
|
||||
# currently disabled as files are large and we do not support rotation
|
||||
_ENABLE_PREPROCESSED_PLAYER_CACHE = False
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._available = True
|
||||
self.ejs_settings = self.ie.get_param('extractor_args', {}).get('youtube-ejs', {})
|
||||
|
||||
# Note: The following 3 args are for developer use only & intentionally not documented.
|
||||
# - dev: bypasses verification of script hashes and versions.
|
||||
# - repo: use a custom GitHub repository to fetch web script from.
|
||||
# - script_version: use a custom script version.
|
||||
# E.g. --extractor-args "youtube-ejs:dev=true;script_version=0.1.4"
|
||||
|
||||
self.is_dev = self.ejs_setting('dev', ['false'])[0] == 'true'
|
||||
if self.is_dev:
|
||||
self.report_dev_option('You have enabled dev mode for EJS JCP Providers.')
|
||||
|
||||
custom_repo = self.ejs_setting('repo', [None])[0]
|
||||
if custom_repo:
|
||||
self.report_dev_option(f'You have set a custom GitHub repository for EJS JCP Providers ({custom_repo}).')
|
||||
self._REPOSITORY = custom_repo
|
||||
|
||||
custom_version = self.ejs_setting('script_version', [None])[0]
|
||||
if custom_version:
|
||||
self.report_dev_option(f'You have set a custom EJS script version for EJS JCP Providers ({custom_version}).')
|
||||
self._SCRIPT_VERSION = custom_version
|
||||
|
||||
def ejs_setting(self, key, *args, **kwargs):
|
||||
return configuration_arg(self.ejs_settings, key, *args, **kwargs)
|
||||
|
||||
def report_dev_option(self, message: str):
|
||||
self.ie.report_warning(
|
||||
f'{message} '
|
||||
f'This is a developer option intended for debugging. \n'
|
||||
' If you experience any issues while using this option, '
|
||||
f'{self.ie._downloader._format_err("DO NOT", self.ie._downloader.Styles.ERROR)} open a bug report', only_once=True)
|
||||
|
||||
def _run_js_runtime(self, stdin: str, /) -> str:
|
||||
"""To be implemented by subclasses"""
|
||||
raise NotImplementedError
|
||||
|
||||
def _real_bulk_solve(self, /, requests: list[JsChallengeRequest]):
|
||||
grouped: dict[str, list[JsChallengeRequest]] = collections.defaultdict(list)
|
||||
for request in requests:
|
||||
grouped[request.input.player_url].append(request)
|
||||
|
||||
for player_url, grouped_requests in grouped.items():
|
||||
player = None
|
||||
if self._ENABLE_PREPROCESSED_PLAYER_CACHE:
|
||||
player = self.ie.cache.load(self._CACHE_SECTION, f'player:{player_url}')
|
||||
|
||||
if player:
|
||||
cached = True
|
||||
else:
|
||||
cached = False
|
||||
video_id = next((request.video_id for request in grouped_requests), None)
|
||||
player = self._get_player(video_id, player_url)
|
||||
|
||||
# NB: This output belongs after the player request
|
||||
self.logger.info(f'Solving JS challenges using {self.JS_RUNTIME_NAME}')
|
||||
|
||||
stdin = self._construct_stdin(player, cached, grouped_requests)
|
||||
stdout = self._run_js_runtime(stdin)
|
||||
output = json.loads(stdout)
|
||||
if output['type'] == 'error':
|
||||
raise JsChallengeProviderError(output['error'])
|
||||
|
||||
if self._ENABLE_PREPROCESSED_PLAYER_CACHE and (preprocessed := output.get('preprocessed_player')):
|
||||
self.ie.cache.store(self._CACHE_SECTION, f'player:{player_url}', preprocessed)
|
||||
|
||||
for request, response_data in zip(grouped_requests, output['responses'], strict=True):
|
||||
if response_data['type'] == 'error':
|
||||
yield JsChallengeProviderResponse(request, None, response_data['error'])
|
||||
else:
|
||||
yield JsChallengeProviderResponse(request, JsChallengeResponse(request.type, (
|
||||
NChallengeOutput(response_data['data']) if request.type is JsChallengeType.N
|
||||
else SigChallengeOutput(response_data['data']))))
|
||||
|
||||
def _construct_stdin(self, player: str, preprocessed: bool, requests: list[JsChallengeRequest], /) -> str:
|
||||
json_requests = [{
|
||||
'type': request.type.value,
|
||||
'challenges': request.input.challenges,
|
||||
} for request in requests]
|
||||
data = {
|
||||
'type': 'preprocessed',
|
||||
'preprocessed_player': player,
|
||||
'requests': json_requests,
|
||||
} if preprocessed else {
|
||||
'type': 'player',
|
||||
'player': player,
|
||||
'requests': json_requests,
|
||||
'output_preprocessed': True,
|
||||
}
|
||||
return f'''\
|
||||
{self._lib_script.code}
|
||||
Object.assign(globalThis, lib);
|
||||
{self._core_script.code}
|
||||
console.log(JSON.stringify(jsc({json.dumps(data)})));
|
||||
'''
|
||||
|
||||
# region: challenge solver script
|
||||
|
||||
@functools.cached_property
|
||||
def _lib_script(self, /):
|
||||
return self._get_script(ScriptType.LIB)
|
||||
|
||||
@functools.cached_property
|
||||
def _core_script(self, /):
|
||||
return self._get_script(ScriptType.CORE)
|
||||
|
||||
def _get_script(self, script_type: ScriptType, /) -> Script:
|
||||
skipped_components: list[_SkippedComponent] = []
|
||||
for _, from_source in self._iter_script_sources():
|
||||
script = from_source(script_type)
|
||||
if not script:
|
||||
continue
|
||||
if isinstance(script, _SkippedComponent):
|
||||
skipped_components.append(script)
|
||||
continue
|
||||
if not self.is_dev:
|
||||
# Matching patch version is expected to have same hash
|
||||
if version_tuple(script.version, lenient=True)[:2] != version_tuple(self._SCRIPT_VERSION, lenient=True)[:2]:
|
||||
self.logger.warning(
|
||||
f'Challenge solver {script_type.value} script version {script.version} '
|
||||
f'is not supported (source: {script.source.value}, variant: {script.variant}, supported version: {self._SCRIPT_VERSION})')
|
||||
if script.source is ScriptSource.CACHE:
|
||||
self.logger.debug('Clearing outdated cached script')
|
||||
self.ie.cache.store(self._CACHE_SECTION, script_type.value, None)
|
||||
continue
|
||||
script_hashes = self._ALLOWED_HASHES[script.type].get(script.variant, [])
|
||||
if script_hashes and script.hash not in script_hashes:
|
||||
self.logger.warning(
|
||||
f'Hash mismatch on challenge solver {script.type.value} script '
|
||||
f'(source: {script.source.value}, variant: {script.variant}, hash: {script.hash})!{provider_bug_report_message(self)}')
|
||||
if script.source is ScriptSource.CACHE:
|
||||
self.logger.debug('Clearing invalid cached script')
|
||||
self.ie.cache.store(self._CACHE_SECTION, script_type.value, None)
|
||||
continue
|
||||
self.logger.debug(
|
||||
f'Using challenge solver {script.type.value} script v{script.version} '
|
||||
f'(source: {script.source.value}, variant: {script.variant.value})')
|
||||
break
|
||||
|
||||
else:
|
||||
self._available = False
|
||||
raise JsChallengeProviderRejectedRequest(
|
||||
f'No usable challenge solver {script_type.value} script available',
|
||||
_skipped_components=skipped_components or None,
|
||||
)
|
||||
|
||||
return script
|
||||
|
||||
def _iter_script_sources(self) -> Generator[tuple[ScriptSource, Callable[[ScriptType], Script | None]]]:
|
||||
yield from [
|
||||
(ScriptSource.PYPACKAGE, self._pypackage_source),
|
||||
(ScriptSource.CACHE, self._cached_source),
|
||||
(ScriptSource.BUILTIN, self._builtin_source),
|
||||
(ScriptSource.WEB, self._web_release_source)]
|
||||
|
||||
def _pypackage_source(self, script_type: ScriptType, /) -> Script | None:
|
||||
if not _has_ejs:
|
||||
return None
|
||||
try:
|
||||
code = yt_dlp_ejs.yt.solver.core() if script_type is ScriptType.CORE else yt_dlp_ejs.yt.solver.lib()
|
||||
except Exception as e:
|
||||
self.logger.warning(
|
||||
f'Failed to load challenge solver {script_type.value} script from python package: {e}{provider_bug_report_message(self)}')
|
||||
return None
|
||||
return Script(script_type, ScriptVariant.MINIFIED, ScriptSource.PYPACKAGE, yt_dlp_ejs.version, code)
|
||||
|
||||
def _cached_source(self, script_type: ScriptType, /) -> Script | None:
|
||||
if data := self.ie.cache.load(self._CACHE_SECTION, script_type.value):
|
||||
return Script(script_type, ScriptVariant(data['variant']), ScriptSource.CACHE, data['version'], data['code'])
|
||||
return None
|
||||
|
||||
def _builtin_source(self, script_type: ScriptType, /) -> Script | None:
|
||||
error_hook = lambda _: self.logger.warning(
|
||||
f'Failed to read builtin challenge solver {script_type.value} script{provider_bug_report_message(self)}')
|
||||
code = vendor.load_script(
|
||||
self._SCRIPT_FILENAMES[script_type], error_hook=error_hook)
|
||||
if code:
|
||||
return Script(script_type, ScriptVariant.UNMINIFIED, ScriptSource.BUILTIN, self._SCRIPT_VERSION, code)
|
||||
return None
|
||||
|
||||
def _web_release_source(self, script_type: ScriptType, /):
|
||||
if 'ejs:github' not in (self.ie.get_param('remote_components') or ()):
|
||||
return self._skip_component('ejs:github')
|
||||
url = f'https://github.com/{self._REPOSITORY}/releases/download/{self._SCRIPT_VERSION}/{self._MIN_SCRIPT_FILENAMES[script_type]}'
|
||||
if code := self.ie._download_webpage_with_retries(
|
||||
url, None, f'[{self.logger.prefix}] Downloading challenge solver {script_type.value} script from {url}',
|
||||
f'[{self.logger.prefix}] Failed to download challenge solver {script_type.value} script', fatal=False,
|
||||
):
|
||||
self.ie.cache.store(self._CACHE_SECTION, script_type.value, {
|
||||
'version': self._SCRIPT_VERSION,
|
||||
'variant': ScriptVariant.MINIFIED.value,
|
||||
'code': code,
|
||||
})
|
||||
return Script(script_type, ScriptVariant.MINIFIED, ScriptSource.WEB, self._SCRIPT_VERSION, code)
|
||||
return None
|
||||
|
||||
# endregion: challenge solver script
|
||||
|
||||
@property
|
||||
def runtime_info(self) -> JsRuntimeInfo | None:
|
||||
runtime = self.ie._downloader._js_runtimes.get(self.JS_RUNTIME_NAME)
|
||||
if not runtime or not runtime.info or not runtime.info.supported:
|
||||
return None
|
||||
return runtime.info
|
||||
|
||||
def is_available(self, /) -> bool:
|
||||
if not self.runtime_info:
|
||||
return False
|
||||
return self._available
|
||||
|
||||
def _skip_component(self, component: str, /):
|
||||
return _SkippedComponent(component, self.JS_RUNTIME_NAME)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class _SkippedComponent:
|
||||
component: str
|
||||
runtime: str
|
||||
@@ -0,0 +1,70 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import EJSBaseJCP
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeRequest,
|
||||
register_preference,
|
||||
register_provider,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider
|
||||
from yt_dlp.utils import Popen
|
||||
|
||||
|
||||
@register_provider
|
||||
class NodeJCP(EJSBaseJCP, BuiltinIEContentProvider):
|
||||
PROVIDER_NAME = 'node'
|
||||
JS_RUNTIME_NAME = 'node'
|
||||
|
||||
_ARGS = ['-']
|
||||
|
||||
def _run_js_runtime(self, stdin: str, /) -> str:
|
||||
args = []
|
||||
|
||||
if self.ejs_setting('jitless', ['false']) != ['false']:
|
||||
args.append('--v8-flags=--jitless')
|
||||
|
||||
# Node permission flag changed from experimental to stable in v23.5.0
|
||||
if self.runtime_info.version_tuple < (23, 5, 0):
|
||||
args.append('--experimental-permission')
|
||||
args.append('--no-warnings=ExperimentalWarning')
|
||||
else:
|
||||
args.append('--permission')
|
||||
|
||||
cmd = [self.runtime_info.path, *args, *self._ARGS]
|
||||
self.logger.debug(f'Running node: {shlex.join(cmd)}')
|
||||
with Popen(
|
||||
cmd,
|
||||
text=True,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
) as proc:
|
||||
stdout, stderr = proc.communicate_or_kill(stdin)
|
||||
stderr = self._clean_stderr(stderr)
|
||||
if proc.returncode or stderr:
|
||||
msg = f'Error running node process (returncode: {proc.returncode})'
|
||||
if stderr:
|
||||
msg = f'{msg}: {stderr.strip()}'
|
||||
raise JsChallengeProviderError(msg)
|
||||
|
||||
return stdout
|
||||
|
||||
def _clean_stderr(self, stderr):
|
||||
return '\n'.join(
|
||||
line for line in stderr.splitlines()
|
||||
if not (
|
||||
re.match(r'^\[stdin\]:', line)
|
||||
or re.match(r'^var jsc', line)
|
||||
or '(Use `node --trace-uncaught ...` to show where the exception was thrown)' == line
|
||||
or re.match(r'^Node\.js v\d+\.\d+\.\d+$', line)))
|
||||
|
||||
|
||||
@register_preference(NodeJCP)
|
||||
def preference(provider: JsChallengeProvider, requests: list[JsChallengeRequest]) -> int:
|
||||
return 900
|
||||
@@ -0,0 +1,59 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import shlex
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import EJSBaseJCP
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeRequest,
|
||||
register_preference,
|
||||
register_provider,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider
|
||||
from yt_dlp.utils import Popen
|
||||
|
||||
|
||||
@register_provider
|
||||
class QuickJSJCP(EJSBaseJCP, BuiltinIEContentProvider):
|
||||
PROVIDER_NAME = 'quickjs'
|
||||
JS_RUNTIME_NAME = 'quickjs'
|
||||
|
||||
def _run_js_runtime(self, stdin: str, /) -> str:
|
||||
if self.runtime_info.name == 'quickjs-ng':
|
||||
self.logger.warning('QuickJS-NG is missing some optimizations making this very slow. Consider using upstream QuickJS instead.')
|
||||
elif self.runtime_info.version_tuple < (2025, 4, 26):
|
||||
self.logger.warning('Older QuickJS versions are missing optimizations making this very slow. Consider upgrading.')
|
||||
|
||||
# QuickJS does not support reading from stdin, so we have to use a temp file
|
||||
temp_file = tempfile.NamedTemporaryFile(mode='w', suffix='.js', delete=False, encoding='utf-8')
|
||||
try:
|
||||
temp_file.write(stdin)
|
||||
temp_file.close()
|
||||
cmd = [self.runtime_info.path, '--script', temp_file.name]
|
||||
self.logger.debug(f'Running QuickJS: {shlex.join(cmd)}')
|
||||
with Popen(
|
||||
cmd,
|
||||
text=True,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
) as proc:
|
||||
stdout, stderr = proc.communicate_or_kill()
|
||||
if proc.returncode or stderr:
|
||||
msg = f'Error running QuickJS process (returncode: {proc.returncode})'
|
||||
if stderr:
|
||||
msg = f'{msg}: {stderr.strip()}'
|
||||
raise JsChallengeProviderError(msg)
|
||||
finally:
|
||||
pathlib.Path(temp_file.name).unlink(missing_ok=True)
|
||||
|
||||
return stdout
|
||||
|
||||
|
||||
@register_preference(QuickJSJCP)
|
||||
def preference(provider: JsChallengeProvider, requests: list[JsChallengeRequest]) -> int:
|
||||
return 850
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
import importlib.resources
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.vendor._info import HASHES, VERSION
|
||||
|
||||
__all__ = ['HASHES', 'VERSION', 'load_script']
|
||||
|
||||
|
||||
def load_script(filename, error_hook=None):
|
||||
file = importlib.resources.files(__package__) / filename
|
||||
if file.is_file():
|
||||
try:
|
||||
return file.read_text(encoding='utf-8')
|
||||
except (OSError, FileNotFoundError, ModuleNotFoundError) as e:
|
||||
if error_hook:
|
||||
error_hook(e)
|
||||
return None
|
||||
return None
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
# This file is generated by devscripts/update_ejs.py. DO NOT MODIFY!
|
||||
|
||||
VERSION = '0.3.2'
|
||||
HASHES = {
|
||||
'yt.solver.bun.lib.js': '6ff45e94de9f0ea936a183c48173cfa9ce526ee4b7544cd556428427c1dd53c8073ef0174e79b320252bf0e7c64b0032cc1cf9c4358f3fda59033b7caa01c241',
|
||||
'yt.solver.core.js': '0cd96b2d3f319dfa62cae689efa7d930ef1706e95f5921794db5089b2262957ec0a17d73938d8975ea35d0309cbfb4c8e4418d5e219837215eee242890c8b64d',
|
||||
'yt.solver.core.min.js': '370d627703002b4a73b10027702734a3de9484f6b56b739942be1dc2b60fee49dee2aa86ed117d1c8ae1ac55181d326481f1fe2e2e8d5211154d48e2a55dac51',
|
||||
'yt.solver.deno.lib.js': '9c8ee3ab6c23e443a5a951e3ac73c6b8c1c8fb34335e7058a07bf99d349be5573611de00536dcd03ecd3cf34014c4e9b536081de37af3637c5390c6a6fd6a0f0',
|
||||
'yt.solver.lib.js': '1ee3753a8222fc855f5c39db30a9ccbb7967dbe1fb810e86dc9a89aa073a0907f294c720e9b65427d560a35aa1ce6af19ef854d9126a05ca00afe03f72047733',
|
||||
'yt.solver.lib.min.js': '8420c259ad16e99ce004e4651ac1bcabb53b4457bf5668a97a9359be9a998a789fee8ab124ee17f91a2ea8fd84e0f2b2fc8eabcaf0b16a186ba734cf422ad053',
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
/*!
|
||||
* SPDX-License-Identifier: Unlicense
|
||||
* This file was automatically generated by https://github.com/yt-dlp/ejs
|
||||
*/
|
||||
const lib = {
|
||||
meriyah: await import('[email protected]'),
|
||||
astring: await import('[email protected]'),
|
||||
};
|
||||
export { lib };
|
||||
+550
@@ -0,0 +1,550 @@
|
||||
/*!
|
||||
* SPDX-License-Identifier: Unlicense
|
||||
* This file was automatically generated by https://github.com/yt-dlp/ejs
|
||||
*/
|
||||
var jsc = (function (meriyah, astring) {
|
||||
'use strict';
|
||||
function matchesStructure(obj, structure) {
|
||||
if (Array.isArray(structure)) {
|
||||
if (!Array.isArray(obj)) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
structure.length === obj.length &&
|
||||
structure.every((value, index) => matchesStructure(obj[index], value))
|
||||
);
|
||||
}
|
||||
if (typeof structure === 'object') {
|
||||
if (!obj) {
|
||||
return !structure;
|
||||
}
|
||||
if ('or' in structure) {
|
||||
return structure.or.some((node) => matchesStructure(obj, node));
|
||||
}
|
||||
if ('anykey' in structure && Array.isArray(structure.anykey)) {
|
||||
const haystack = Array.isArray(obj) ? obj : Object.values(obj);
|
||||
return structure.anykey.every((value) =>
|
||||
haystack.some((el) => matchesStructure(el, value)),
|
||||
);
|
||||
}
|
||||
for (const [key, value] of Object.entries(structure)) {
|
||||
if (!matchesStructure(obj[key], value)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return structure === obj;
|
||||
}
|
||||
function isOneOf(value, ...of) {
|
||||
return of.includes(value);
|
||||
}
|
||||
function _optionalChain$2(ops) {
|
||||
let lastAccessLHS = undefined;
|
||||
let value = ops[0];
|
||||
let i = 1;
|
||||
while (i < ops.length) {
|
||||
const op = ops[i];
|
||||
const fn = ops[i + 1];
|
||||
i += 2;
|
||||
if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) {
|
||||
return undefined;
|
||||
}
|
||||
if (op === 'access' || op === 'optionalAccess') {
|
||||
lastAccessLHS = value;
|
||||
value = fn(value);
|
||||
} else if (op === 'call' || op === 'optionalCall') {
|
||||
value = fn((...args) => value.call(lastAccessLHS, ...args));
|
||||
lastAccessLHS = undefined;
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
const logicalExpression = {
|
||||
type: 'ExpressionStatement',
|
||||
expression: {
|
||||
type: 'LogicalExpression',
|
||||
left: { type: 'Identifier' },
|
||||
right: {
|
||||
type: 'SequenceExpression',
|
||||
expressions: [
|
||||
{
|
||||
type: 'AssignmentExpression',
|
||||
left: { type: 'Identifier' },
|
||||
operator: '=',
|
||||
right: {
|
||||
type: 'CallExpression',
|
||||
callee: { type: 'Identifier' },
|
||||
arguments: {
|
||||
or: [
|
||||
[
|
||||
{ type: 'Literal' },
|
||||
{
|
||||
type: 'CallExpression',
|
||||
callee: {
|
||||
type: 'Identifier',
|
||||
name: 'decodeURIComponent',
|
||||
},
|
||||
arguments: [{ type: 'Identifier' }],
|
||||
optional: false,
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
type: 'CallExpression',
|
||||
callee: {
|
||||
type: 'Identifier',
|
||||
name: 'decodeURIComponent',
|
||||
},
|
||||
arguments: [{ type: 'Identifier' }],
|
||||
optional: false,
|
||||
},
|
||||
],
|
||||
],
|
||||
},
|
||||
optional: false,
|
||||
},
|
||||
},
|
||||
{ type: 'CallExpression' },
|
||||
],
|
||||
},
|
||||
operator: '&&',
|
||||
},
|
||||
};
|
||||
const identifier$1 = {
|
||||
or: [
|
||||
{
|
||||
type: 'ExpressionStatement',
|
||||
expression: {
|
||||
type: 'AssignmentExpression',
|
||||
operator: '=',
|
||||
left: { type: 'Identifier' },
|
||||
right: { type: 'FunctionExpression', params: [{}, {}, {}] },
|
||||
},
|
||||
},
|
||||
{ type: 'FunctionDeclaration', params: [{}, {}, {}] },
|
||||
{
|
||||
type: 'VariableDeclaration',
|
||||
declarations: {
|
||||
anykey: [
|
||||
{
|
||||
type: 'VariableDeclarator',
|
||||
init: { type: 'FunctionExpression', params: [{}, {}, {}] },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
function extract$1(node) {
|
||||
if (!matchesStructure(node, identifier$1)) {
|
||||
return null;
|
||||
}
|
||||
let block;
|
||||
if (
|
||||
node.type === 'ExpressionStatement' &&
|
||||
node.expression.type === 'AssignmentExpression' &&
|
||||
node.expression.right.type === 'FunctionExpression'
|
||||
) {
|
||||
block = node.expression.right.body;
|
||||
} else if (node.type === 'VariableDeclaration') {
|
||||
for (const decl of node.declarations) {
|
||||
if (
|
||||
decl.type === 'VariableDeclarator' &&
|
||||
_optionalChain$2([
|
||||
decl,
|
||||
'access',
|
||||
(_) => _.init,
|
||||
'optionalAccess',
|
||||
(_2) => _2.type,
|
||||
]) === 'FunctionExpression' &&
|
||||
_optionalChain$2([
|
||||
decl,
|
||||
'access',
|
||||
(_3) => _3.init,
|
||||
'optionalAccess',
|
||||
(_4) => _4.params,
|
||||
'access',
|
||||
(_5) => _5.length,
|
||||
]) === 3
|
||||
) {
|
||||
block = decl.init.body;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (node.type === 'FunctionDeclaration') {
|
||||
block = node.body;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
const relevantExpression = _optionalChain$2([
|
||||
block,
|
||||
'optionalAccess',
|
||||
(_6) => _6.body,
|
||||
'access',
|
||||
(_7) => _7.at,
|
||||
'call',
|
||||
(_8) => _8(-2),
|
||||
]);
|
||||
if (!matchesStructure(relevantExpression, logicalExpression)) {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
_optionalChain$2([
|
||||
relevantExpression,
|
||||
'optionalAccess',
|
||||
(_9) => _9.type,
|
||||
]) !== 'ExpressionStatement' ||
|
||||
relevantExpression.expression.type !== 'LogicalExpression' ||
|
||||
relevantExpression.expression.right.type !== 'SequenceExpression' ||
|
||||
relevantExpression.expression.right.expressions[0].type !==
|
||||
'AssignmentExpression'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const call = relevantExpression.expression.right.expressions[0].right;
|
||||
if (call.type !== 'CallExpression' || call.callee.type !== 'Identifier') {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
type: 'ArrowFunctionExpression',
|
||||
params: [{ type: 'Identifier', name: 'sig' }],
|
||||
body: {
|
||||
type: 'CallExpression',
|
||||
callee: { type: 'Identifier', name: call.callee.name },
|
||||
arguments:
|
||||
call.arguments.length === 1
|
||||
? [{ type: 'Identifier', name: 'sig' }]
|
||||
: [call.arguments[0], { type: 'Identifier', name: 'sig' }],
|
||||
optional: false,
|
||||
},
|
||||
async: false,
|
||||
expression: false,
|
||||
generator: false,
|
||||
};
|
||||
}
|
||||
function _optionalChain$1(ops) {
|
||||
let lastAccessLHS = undefined;
|
||||
let value = ops[0];
|
||||
let i = 1;
|
||||
while (i < ops.length) {
|
||||
const op = ops[i];
|
||||
const fn = ops[i + 1];
|
||||
i += 2;
|
||||
if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) {
|
||||
return undefined;
|
||||
}
|
||||
if (op === 'access' || op === 'optionalAccess') {
|
||||
lastAccessLHS = value;
|
||||
value = fn(value);
|
||||
} else if (op === 'call' || op === 'optionalCall') {
|
||||
value = fn((...args) => value.call(lastAccessLHS, ...args));
|
||||
lastAccessLHS = undefined;
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
const identifier = {
|
||||
or: [
|
||||
{
|
||||
type: 'VariableDeclaration',
|
||||
kind: 'var',
|
||||
declarations: {
|
||||
anykey: [
|
||||
{
|
||||
type: 'VariableDeclarator',
|
||||
id: { type: 'Identifier' },
|
||||
init: {
|
||||
type: 'ArrayExpression',
|
||||
elements: [{ type: 'Identifier' }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'ExpressionStatement',
|
||||
expression: {
|
||||
type: 'AssignmentExpression',
|
||||
left: { type: 'Identifier' },
|
||||
operator: '=',
|
||||
right: {
|
||||
type: 'ArrayExpression',
|
||||
elements: [{ type: 'Identifier' }],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const catchBlockBody = [
|
||||
{
|
||||
type: 'ReturnStatement',
|
||||
argument: {
|
||||
type: 'BinaryExpression',
|
||||
left: {
|
||||
type: 'MemberExpression',
|
||||
object: { type: 'Identifier' },
|
||||
computed: true,
|
||||
property: { type: 'Literal' },
|
||||
optional: false,
|
||||
},
|
||||
right: { type: 'Identifier' },
|
||||
operator: '+',
|
||||
},
|
||||
},
|
||||
];
|
||||
function extract(node) {
|
||||
if (!matchesStructure(node, identifier)) {
|
||||
let name = null;
|
||||
let block = null;
|
||||
switch (node.type) {
|
||||
case 'ExpressionStatement': {
|
||||
if (
|
||||
node.expression.type === 'AssignmentExpression' &&
|
||||
node.expression.left.type === 'Identifier' &&
|
||||
node.expression.right.type === 'FunctionExpression' &&
|
||||
node.expression.right.params.length === 1
|
||||
) {
|
||||
name = node.expression.left.name;
|
||||
block = node.expression.right.body;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'FunctionDeclaration': {
|
||||
if (node.params.length === 1) {
|
||||
name = _optionalChain$1([
|
||||
node,
|
||||
'access',
|
||||
(_) => _.id,
|
||||
'optionalAccess',
|
||||
(_2) => _2.name,
|
||||
]);
|
||||
block = node.body;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!block || !name) {
|
||||
return null;
|
||||
}
|
||||
const tryNode = block.body.at(-2);
|
||||
if (
|
||||
_optionalChain$1([tryNode, 'optionalAccess', (_3) => _3.type]) !==
|
||||
'TryStatement' ||
|
||||
_optionalChain$1([
|
||||
tryNode,
|
||||
'access',
|
||||
(_4) => _4.handler,
|
||||
'optionalAccess',
|
||||
(_5) => _5.type,
|
||||
]) !== 'CatchClause'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const catchBody = tryNode.handler.body.body;
|
||||
if (matchesStructure(catchBody, catchBlockBody)) {
|
||||
return makeSolverFuncFromName(name);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
if (node.type === 'VariableDeclaration') {
|
||||
for (const declaration of node.declarations) {
|
||||
if (
|
||||
declaration.type !== 'VariableDeclarator' ||
|
||||
!declaration.init ||
|
||||
declaration.init.type !== 'ArrayExpression' ||
|
||||
declaration.init.elements.length !== 1
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const [firstElement] = declaration.init.elements;
|
||||
if (firstElement && firstElement.type === 'Identifier') {
|
||||
return makeSolverFuncFromName(firstElement.name);
|
||||
}
|
||||
}
|
||||
} else if (node.type === 'ExpressionStatement') {
|
||||
const expr = node.expression;
|
||||
if (
|
||||
expr.type === 'AssignmentExpression' &&
|
||||
expr.left.type === 'Identifier' &&
|
||||
expr.operator === '=' &&
|
||||
expr.right.type === 'ArrayExpression' &&
|
||||
expr.right.elements.length === 1
|
||||
) {
|
||||
const [firstElement] = expr.right.elements;
|
||||
if (firstElement && firstElement.type === 'Identifier') {
|
||||
return makeSolverFuncFromName(firstElement.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function makeSolverFuncFromName(name) {
|
||||
return {
|
||||
type: 'ArrowFunctionExpression',
|
||||
params: [{ type: 'Identifier', name: 'n' }],
|
||||
body: {
|
||||
type: 'CallExpression',
|
||||
callee: { type: 'Identifier', name: name },
|
||||
arguments: [{ type: 'Identifier', name: 'n' }],
|
||||
optional: false,
|
||||
},
|
||||
async: false,
|
||||
expression: false,
|
||||
generator: false,
|
||||
};
|
||||
}
|
||||
const setupNodes = meriyah.parse(
|
||||
`\nif (typeof globalThis.XMLHttpRequest === "undefined") {\n globalThis.XMLHttpRequest = { prototype: {} };\n}\nconst window = Object.create(null);\nif (typeof URL === "undefined") {\n window.location = {\n hash: "",\n host: "www.youtube.com",\n hostname: "www.youtube.com",\n href: "https://www.youtube.com/watch?v=yt-dlp-wins",\n origin: "https://www.youtube.com",\n password: "",\n pathname: "/watch",\n port: "",\n protocol: "https:",\n search: "?v=yt-dlp-wins",\n username: "",\n };\n} else {\n window.location = new URL("https://www.youtube.com/watch?v=yt-dlp-wins");\n}\nif (typeof globalThis.document === "undefined") {\n globalThis.document = Object.create(null);\n}\nif (typeof globalThis.navigator === "undefined") {\n globalThis.navigator = Object.create(null);\n}\nif (typeof globalThis.self === "undefined") {\n globalThis.self = globalThis;\n}\n`,
|
||||
).body;
|
||||
function _optionalChain(ops) {
|
||||
let lastAccessLHS = undefined;
|
||||
let value = ops[0];
|
||||
let i = 1;
|
||||
while (i < ops.length) {
|
||||
const op = ops[i];
|
||||
const fn = ops[i + 1];
|
||||
i += 2;
|
||||
if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) {
|
||||
return undefined;
|
||||
}
|
||||
if (op === 'access' || op === 'optionalAccess') {
|
||||
lastAccessLHS = value;
|
||||
value = fn(value);
|
||||
} else if (op === 'call' || op === 'optionalCall') {
|
||||
value = fn((...args) => value.call(lastAccessLHS, ...args));
|
||||
lastAccessLHS = undefined;
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
function preprocessPlayer(data) {
|
||||
const ast = meriyah.parse(data);
|
||||
const body = ast.body;
|
||||
const block = (() => {
|
||||
switch (body.length) {
|
||||
case 1: {
|
||||
const func = body[0];
|
||||
if (
|
||||
_optionalChain([func, 'optionalAccess', (_) => _.type]) ===
|
||||
'ExpressionStatement' &&
|
||||
func.expression.type === 'CallExpression' &&
|
||||
func.expression.callee.type === 'MemberExpression' &&
|
||||
func.expression.callee.object.type === 'FunctionExpression'
|
||||
) {
|
||||
return func.expression.callee.object.body;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
const func = body[1];
|
||||
if (
|
||||
_optionalChain([func, 'optionalAccess', (_2) => _2.type]) ===
|
||||
'ExpressionStatement' &&
|
||||
func.expression.type === 'CallExpression' &&
|
||||
func.expression.callee.type === 'FunctionExpression'
|
||||
) {
|
||||
const block = func.expression.callee.body;
|
||||
block.body.splice(0, 1);
|
||||
return block;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
throw 'unexpected structure';
|
||||
})();
|
||||
const found = { n: [], sig: [] };
|
||||
const plainExpressions = block.body.filter((node) => {
|
||||
const n = extract(node);
|
||||
if (n) {
|
||||
found.n.push(n);
|
||||
}
|
||||
const sig = extract$1(node);
|
||||
if (sig) {
|
||||
found.sig.push(sig);
|
||||
}
|
||||
if (node.type === 'ExpressionStatement') {
|
||||
if (node.expression.type === 'AssignmentExpression') {
|
||||
return true;
|
||||
}
|
||||
return node.expression.type === 'Literal';
|
||||
}
|
||||
return true;
|
||||
});
|
||||
block.body = plainExpressions;
|
||||
for (const [name, options] of Object.entries(found)) {
|
||||
const unique = new Set(options.map((x) => JSON.stringify(x)));
|
||||
if (unique.size !== 1) {
|
||||
const message = `found ${unique.size} ${name} function possibilities`;
|
||||
throw (
|
||||
message +
|
||||
(unique.size
|
||||
? `: ${options.map((x) => astring.generate(x)).join(', ')}`
|
||||
: '')
|
||||
);
|
||||
}
|
||||
plainExpressions.push({
|
||||
type: 'ExpressionStatement',
|
||||
expression: {
|
||||
type: 'AssignmentExpression',
|
||||
operator: '=',
|
||||
left: {
|
||||
type: 'MemberExpression',
|
||||
computed: false,
|
||||
object: { type: 'Identifier', name: '_result' },
|
||||
property: { type: 'Identifier', name: name },
|
||||
},
|
||||
right: options[0],
|
||||
},
|
||||
});
|
||||
}
|
||||
ast.body.splice(0, 0, ...setupNodes);
|
||||
return astring.generate(ast);
|
||||
}
|
||||
function getFromPrepared(code) {
|
||||
const resultObj = { n: null, sig: null };
|
||||
Function('_result', code)(resultObj);
|
||||
return resultObj;
|
||||
}
|
||||
function main(input) {
|
||||
const preprocessedPlayer =
|
||||
input.type === 'player'
|
||||
? preprocessPlayer(input.player)
|
||||
: input.preprocessed_player;
|
||||
const solvers = getFromPrepared(preprocessedPlayer);
|
||||
const responses = input.requests.map((input) => {
|
||||
if (!isOneOf(input.type, 'n', 'sig')) {
|
||||
return { type: 'error', error: `Unknown request type: ${input.type}` };
|
||||
}
|
||||
const solver = solvers[input.type];
|
||||
if (!solver) {
|
||||
return {
|
||||
type: 'error',
|
||||
error: `Failed to extract ${input.type} function`,
|
||||
};
|
||||
}
|
||||
try {
|
||||
return {
|
||||
type: 'result',
|
||||
data: Object.fromEntries(
|
||||
input.challenges.map((challenge) => [challenge, solver(challenge)]),
|
||||
),
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
type: 'error',
|
||||
error:
|
||||
error instanceof Error
|
||||
? `${error.message}\n${error.stack}`
|
||||
: `${error}`,
|
||||
};
|
||||
}
|
||||
});
|
||||
const output = { type: 'result', responses: responses };
|
||||
if (input.type === 'player' && input.output_preprocessed) {
|
||||
output.preprocessed_player = preprocessedPlayer;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
return main;
|
||||
})(meriyah, astring);
|
||||
Vendored
+9
@@ -0,0 +1,9 @@
|
||||
/*!
|
||||
* SPDX-License-Identifier: Unlicense
|
||||
* This file was automatically generated by https://github.com/yt-dlp/ejs
|
||||
*/
|
||||
const lib = {
|
||||
meriyah: await import('npm:[email protected]'),
|
||||
astring: await import('npm:[email protected]'),
|
||||
};
|
||||
export { lib };
|
||||
@@ -0,0 +1,287 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import dataclasses
|
||||
import typing
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import _EJS_WIKI_URL
|
||||
from yt_dlp.extractor.youtube.jsc._registry import (
|
||||
_jsc_preferences,
|
||||
_jsc_providers,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeProviderRejectedRequest,
|
||||
JsChallengeProviderResponse,
|
||||
JsChallengeRequest,
|
||||
JsChallengeResponse,
|
||||
JsChallengeType,
|
||||
NChallengeInput,
|
||||
NChallengeOutput,
|
||||
SigChallengeInput,
|
||||
SigChallengeOutput,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._director import YoutubeIEContentProviderLogger, provider_display_list
|
||||
from yt_dlp.extractor.youtube.pot._provider import (
|
||||
IEContentProviderLogger,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot.provider import (
|
||||
provider_bug_report_message,
|
||||
)
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import _SkippedComponent
|
||||
from yt_dlp.extractor.youtube.jsc.provider import Preference as JsChallengePreference
|
||||
|
||||
|
||||
class JsChallengeRequestDirector:
|
||||
|
||||
def __init__(self, logger: IEContentProviderLogger):
|
||||
self.providers: dict[str, JsChallengeProvider] = {}
|
||||
self.preferences: list[JsChallengePreference] = []
|
||||
self.logger = logger
|
||||
|
||||
def register_provider(self, provider: JsChallengeProvider):
|
||||
self.providers[provider.PROVIDER_KEY] = provider
|
||||
|
||||
def register_preference(self, preference: JsChallengePreference):
|
||||
self.preferences.append(preference)
|
||||
|
||||
def _get_providers(self, requests: list[JsChallengeRequest]) -> Iterable[JsChallengeProvider]:
|
||||
"""Sorts available providers by preference, given a request"""
|
||||
preferences = {
|
||||
provider: sum(pref(provider, requests) for pref in self.preferences)
|
||||
for provider in self.providers.values()
|
||||
}
|
||||
if self.logger.log_level <= self.logger.LogLevel.TRACE:
|
||||
# calling is_available() for every JS Challenge provider upfront may have some overhead
|
||||
self.logger.trace(f'JS Challenge Providers: {provider_display_list(self.providers.values())}')
|
||||
self.logger.trace('JS Challenge Provider preferences for this request: {}'.format(', '.join(
|
||||
f'{provider.PROVIDER_NAME}={pref}' for provider, pref in preferences.items())))
|
||||
|
||||
return (
|
||||
provider for provider in sorted(
|
||||
self.providers.values(), key=preferences.get, reverse=True)
|
||||
if provider.is_available()
|
||||
)
|
||||
|
||||
def _handle_error(self, e: Exception, provider: JsChallengeProvider, requests: list[JsChallengeRequest]):
|
||||
if isinstance(e, JsChallengeProviderRejectedRequest):
|
||||
self.logger.trace(
|
||||
f'JS Challenge Provider "{provider.PROVIDER_NAME}" rejected '
|
||||
f'{"this request" if len(requests) == 1 else f"{len(requests)} requests"}, '
|
||||
f'trying next available provider. Reason: {e}',
|
||||
)
|
||||
elif isinstance(e, JsChallengeProviderError):
|
||||
if len(requests) == 1:
|
||||
self.logger.warning(
|
||||
f'Error solving {requests[0].type.value} challenge request using "{provider.PROVIDER_NAME}" provider: {e}.\n'
|
||||
f' input = {requests[0].input}\n'
|
||||
f' {(provider_bug_report_message(provider, before="") if not e.expected else "")}')
|
||||
else:
|
||||
self.logger.warning(
|
||||
f'Error solving {len(requests)} challenge requests using "{provider.PROVIDER_NAME}" provider: {e}.\n'
|
||||
f' requests = {requests}\n'
|
||||
f' {(provider_bug_report_message(provider, before="") if not e.expected else "")}')
|
||||
else:
|
||||
self.logger.error(
|
||||
f'Unexpected error solving {len(requests)} challenge request(s) using "{provider.PROVIDER_NAME}" provider: {e!r}\n'
|
||||
f' requests = {requests}\n'
|
||||
f' {provider_bug_report_message(provider, before="")}', cause=e)
|
||||
|
||||
def bulk_solve(self, requests: list[JsChallengeRequest]) -> list[tuple[JsChallengeRequest, JsChallengeResponse]]:
|
||||
"""Solves multiple JS Challenges in bulk, returning a list of responses"""
|
||||
if not self.providers:
|
||||
self.logger.trace('No JS Challenge providers registered')
|
||||
return []
|
||||
|
||||
results = []
|
||||
next_requests = requests[:]
|
||||
|
||||
skipped_components = []
|
||||
for provider in self._get_providers(next_requests):
|
||||
if not next_requests:
|
||||
break
|
||||
self.logger.trace(
|
||||
f'Attempting to solve {len(next_requests)} challenges using "{provider.PROVIDER_NAME}" provider')
|
||||
try:
|
||||
for response in provider.bulk_solve([dataclasses.replace(request) for request in next_requests]):
|
||||
if not validate_provider_response(response):
|
||||
self.logger.warning(
|
||||
f'JS Challenge Provider "{provider.PROVIDER_NAME}" returned an invalid response:'
|
||||
f' response = {response!r}\n'
|
||||
f' {provider_bug_report_message(provider, before="")}')
|
||||
continue
|
||||
if response.error:
|
||||
self._handle_error(response.error, provider, [response.request])
|
||||
continue
|
||||
if (vr_msg := validate_response(response.response, response.request)) is not True:
|
||||
self.logger.warning(
|
||||
f'Invalid JS Challenge response received from "{provider.PROVIDER_NAME}" provider: {vr_msg or ""}\n'
|
||||
f' response = {response.response}\n'
|
||||
f' request = {response.request}\n'
|
||||
f' {provider_bug_report_message(provider, before="")}')
|
||||
continue
|
||||
try:
|
||||
next_requests.remove(response.request)
|
||||
except ValueError:
|
||||
self.logger.warning(
|
||||
f'JS Challenge Provider "{provider.PROVIDER_NAME}" returned a response for an unknown request:\n'
|
||||
f' request = {response.request}\n'
|
||||
f' {provider_bug_report_message(provider, before="")}')
|
||||
continue
|
||||
results.append((response.request, response.response))
|
||||
except Exception as e:
|
||||
if isinstance(e, JsChallengeProviderRejectedRequest) and e._skipped_components:
|
||||
skipped_components.extend(e._skipped_components)
|
||||
self._handle_error(e, provider, next_requests)
|
||||
continue
|
||||
|
||||
if skipped_components:
|
||||
self.__report_skipped_components(skipped_components)
|
||||
|
||||
if len(results) != len(requests):
|
||||
self.logger.trace(
|
||||
f'Not all JS Challenges were solved, expected {len(requests)} responses, got {len(results)}')
|
||||
self.logger.trace(f'Unsolved requests: {next_requests}')
|
||||
else:
|
||||
self.logger.trace(f'Solved all {len(requests)} requested JS Challenges')
|
||||
return results
|
||||
|
||||
def __report_skipped_components(self, components: list[_SkippedComponent], /):
|
||||
runtime_components = collections.defaultdict(list)
|
||||
for component in components:
|
||||
runtime_components[component.component].append(component.runtime)
|
||||
for runtimes in runtime_components.values():
|
||||
runtimes.sort()
|
||||
|
||||
description_lookup = {
|
||||
'ejs:npm': 'NPM package',
|
||||
'ejs:github': 'challenge solver script',
|
||||
}
|
||||
|
||||
descriptions = [
|
||||
f'{description_lookup.get(component, component)} ({", ".join(runtimes)})'
|
||||
for component, runtimes in runtime_components.items()
|
||||
if runtimes
|
||||
]
|
||||
flags = [
|
||||
f' --remote-components {f"{component} (recommended)" if component == "ejs:github" else f"{component} "}'
|
||||
for component, runtimes in runtime_components.items()
|
||||
if runtimes
|
||||
]
|
||||
|
||||
def join_parts(parts, joiner):
|
||||
if not parts:
|
||||
return ''
|
||||
if len(parts) == 1:
|
||||
return parts[0]
|
||||
return f'{", ".join(parts[:-1])} {joiner} {parts[-1]}'
|
||||
|
||||
if len(descriptions) == 1:
|
||||
msg = (
|
||||
f'Remote component {descriptions[0]} was skipped. '
|
||||
f'It may be required to solve JS challenges. '
|
||||
f'You can enable the download with {flags[0]}')
|
||||
else:
|
||||
msg = (
|
||||
f'Remote components {join_parts(descriptions, "and")} were skipped. '
|
||||
f'These may be required to solve JS challenges. '
|
||||
f'You can enable these downloads with {join_parts(flags, "or")}, respectively')
|
||||
|
||||
self.logger.warning(f'{msg}. For more information and alternatives, refer to {_EJS_WIKI_URL}')
|
||||
|
||||
def close(self):
|
||||
for provider in self.providers.values():
|
||||
provider.close()
|
||||
|
||||
|
||||
EXTRACTOR_ARG_PREFIX = 'youtubejsc'
|
||||
|
||||
|
||||
def initialize_jsc_director(ie):
|
||||
assert ie._downloader is not None, 'Downloader not set'
|
||||
|
||||
enable_trace = ie._configuration_arg(
|
||||
'jsc_trace', ['false'], ie_key='youtube', casesense=False)[0] == 'true'
|
||||
|
||||
if enable_trace:
|
||||
log_level = IEContentProviderLogger.LogLevel.TRACE
|
||||
elif ie.get_param('verbose', False):
|
||||
log_level = IEContentProviderLogger.LogLevel.DEBUG
|
||||
else:
|
||||
log_level = IEContentProviderLogger.LogLevel.INFO
|
||||
|
||||
def get_provider_logger_and_settings(provider, logger_key):
|
||||
logger_prefix = f'{logger_key}:{provider.PROVIDER_NAME}'
|
||||
extractor_key = f'{EXTRACTOR_ARG_PREFIX}-{provider.PROVIDER_KEY.lower()}'
|
||||
return (
|
||||
YoutubeIEContentProviderLogger(ie, logger_prefix, log_level=log_level),
|
||||
ie.get_param('extractor_args', {}).get(extractor_key, {}))
|
||||
|
||||
director = JsChallengeRequestDirector(
|
||||
logger=YoutubeIEContentProviderLogger(ie, 'jsc', log_level=log_level),
|
||||
)
|
||||
|
||||
ie._downloader.add_close_hook(director.close)
|
||||
|
||||
for provider in _jsc_providers.value.values():
|
||||
logger, settings = get_provider_logger_and_settings(provider, 'jsc')
|
||||
director.register_provider(provider(ie, logger, settings))
|
||||
|
||||
for preference in _jsc_preferences.value:
|
||||
director.register_preference(preference)
|
||||
|
||||
if director.logger.log_level <= director.logger.LogLevel.DEBUG:
|
||||
# calling is_available() for every JS Challenge provider upfront may have some overhead
|
||||
director.logger.debug(f'JS Challenge Providers: {provider_display_list(director.providers.values())}')
|
||||
director.logger.trace(f'Registered {len(director.preferences)} JS Challenge provider preferences')
|
||||
|
||||
return director
|
||||
|
||||
|
||||
def validate_provider_response(response: JsChallengeProviderResponse) -> bool:
|
||||
return (
|
||||
isinstance(response, JsChallengeProviderResponse)
|
||||
and isinstance(response.request, JsChallengeRequest)
|
||||
and (
|
||||
isinstance(response.response, JsChallengeResponse)
|
||||
or (response.error is not None and isinstance(response.error, Exception)))
|
||||
)
|
||||
|
||||
|
||||
def validate_response(response: JsChallengeResponse, request: JsChallengeRequest) -> bool | str:
|
||||
if not isinstance(response, JsChallengeResponse):
|
||||
return 'Response is not a JsChallengeResponse'
|
||||
if request.type == JsChallengeType.N:
|
||||
return validate_nsig_challenge_output(response.output, request.input)
|
||||
else:
|
||||
return validate_sig_challenge_output(response.output, request.input)
|
||||
|
||||
|
||||
def validate_nsig_challenge_output(challenge_output: NChallengeOutput, challenge_input: NChallengeInput) -> bool | str:
|
||||
if not (
|
||||
isinstance(challenge_output, NChallengeOutput)
|
||||
and len(challenge_output.results) == len(challenge_input.challenges)
|
||||
and all(isinstance(k, str) and isinstance(v, str) for k, v in challenge_output.results.items())
|
||||
and all(challenge in challenge_output.results for challenge in challenge_input.challenges)
|
||||
):
|
||||
return 'Invalid NChallengeOutput'
|
||||
|
||||
# Validate n results are valid - if they end with the input challenge then the js function returned with an exception.
|
||||
for challenge, result in challenge_output.results.items():
|
||||
if result.endswith(challenge):
|
||||
return f'n result is invalid for {challenge!r}: {result!r}'
|
||||
return True
|
||||
|
||||
|
||||
def validate_sig_challenge_output(challenge_output: SigChallengeOutput, challenge_input: SigChallengeInput) -> bool:
|
||||
return (
|
||||
isinstance(challenge_output, SigChallengeOutput)
|
||||
and len(challenge_output.results) == len(challenge_input.challenges)
|
||||
and all(isinstance(k, str) and isinstance(v, str) for k, v in challenge_output.results.items())
|
||||
and all(challenge in challenge_output.results for challenge in challenge_input.challenges)
|
||||
) or 'Invalid SigChallengeOutput'
|
||||
@@ -0,0 +1,4 @@
|
||||
from yt_dlp.globals import Indirect
|
||||
|
||||
_jsc_providers = Indirect({})
|
||||
_jsc_preferences = Indirect(set())
|
||||
@@ -0,0 +1,161 @@
|
||||
"""PUBLIC API"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import dataclasses
|
||||
import enum
|
||||
import typing
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._registry import _jsc_preferences, _jsc_providers
|
||||
from yt_dlp.extractor.youtube.pot._provider import (
|
||||
IEContentProvider,
|
||||
IEContentProviderError,
|
||||
register_preference_generic,
|
||||
register_provider_generic,
|
||||
)
|
||||
from yt_dlp.utils import ExtractorError
|
||||
|
||||
__all__ = [
|
||||
'JsChallengeProvider',
|
||||
'JsChallengeProviderError',
|
||||
'JsChallengeProviderRejectedRequest',
|
||||
'JsChallengeProviderResponse',
|
||||
'JsChallengeRequest',
|
||||
'JsChallengeResponse',
|
||||
'JsChallengeType',
|
||||
'NChallengeInput',
|
||||
'NChallengeOutput',
|
||||
'SigChallengeInput',
|
||||
'SigChallengeOutput',
|
||||
'register_preference',
|
||||
'register_provider',
|
||||
]
|
||||
|
||||
|
||||
class JsChallengeType(enum.Enum):
|
||||
N = 'n'
|
||||
SIG = 'sig'
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class JsChallengeRequest:
|
||||
type: JsChallengeType
|
||||
input: NChallengeInput | SigChallengeInput
|
||||
video_id: str | None = None
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class NChallengeInput:
|
||||
player_url: str
|
||||
challenges: list[str] = dataclasses.field(default_factory=list)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class SigChallengeInput:
|
||||
player_url: str
|
||||
challenges: list[str] = dataclasses.field(default_factory=list)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class NChallengeOutput:
|
||||
results: dict[str, str] = dataclasses.field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class SigChallengeOutput:
|
||||
results: dict[str, str] = dataclasses.field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class JsChallengeProviderResponse:
|
||||
request: JsChallengeRequest
|
||||
response: JsChallengeResponse | None = None
|
||||
error: Exception | None = None
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class JsChallengeResponse:
|
||||
type: JsChallengeType
|
||||
output: NChallengeOutput | SigChallengeOutput
|
||||
|
||||
|
||||
class JsChallengeProviderRejectedRequest(IEContentProviderError):
|
||||
"""Reject the JsChallengeRequest (cannot handle the request)"""
|
||||
|
||||
def __init__(self, msg=None, expected: bool = False, *, _skipped_components=None):
|
||||
super().__init__(msg, expected)
|
||||
self._skipped_components = _skipped_components
|
||||
|
||||
|
||||
class JsChallengeProviderError(IEContentProviderError):
|
||||
"""An error occurred while solving the challenge"""
|
||||
|
||||
|
||||
class JsChallengeProvider(IEContentProvider, abc.ABC, suffix='JCP'):
|
||||
|
||||
# Set to None to disable the check
|
||||
_SUPPORTED_TYPES: tuple[JsChallengeType] | None = ()
|
||||
|
||||
def __validate_request(self, request: JsChallengeRequest):
|
||||
if not self.is_available():
|
||||
raise JsChallengeProviderRejectedRequest(f'{self.PROVIDER_NAME} is not available')
|
||||
|
||||
# Validate request using built-in settings
|
||||
if (
|
||||
self._SUPPORTED_TYPES is not None
|
||||
and request.type not in self._SUPPORTED_TYPES
|
||||
):
|
||||
raise JsChallengeProviderRejectedRequest(
|
||||
f'JS Challenge type "{request.type}" is not supported by {self.PROVIDER_NAME}')
|
||||
|
||||
def bulk_solve(self, requests: list[JsChallengeRequest]) -> typing.Generator[JsChallengeProviderResponse, None, None]:
|
||||
"""Solve multiple JS challenges and return the results"""
|
||||
validated_requests = []
|
||||
for request in requests:
|
||||
try:
|
||||
self.__validate_request(request)
|
||||
validated_requests.append(request)
|
||||
except JsChallengeProviderRejectedRequest as e:
|
||||
yield JsChallengeProviderResponse(request=request, error=e)
|
||||
continue
|
||||
yield from self._real_bulk_solve(validated_requests)
|
||||
|
||||
@abc.abstractmethod
|
||||
def _real_bulk_solve(self, requests: list[JsChallengeRequest]) -> typing.Generator[JsChallengeProviderResponse, None, None]:
|
||||
"""Subclasses can override this method to handle bulk solving"""
|
||||
raise NotImplementedError(f'{self.PROVIDER_NAME} does not implement bulk solving')
|
||||
|
||||
def _get_player(self, video_id, player_url):
|
||||
try:
|
||||
return self.ie._load_player(
|
||||
video_id=video_id,
|
||||
player_url=player_url,
|
||||
fatal=True,
|
||||
)
|
||||
except ExtractorError as e:
|
||||
raise JsChallengeProviderError(
|
||||
f'Failed to load player for JS challenge: {e}') from e
|
||||
|
||||
|
||||
def register_provider(provider: type[JsChallengeProvider]):
|
||||
"""Register a JsChallengeProvider class"""
|
||||
return register_provider_generic(
|
||||
provider=provider,
|
||||
base_class=JsChallengeProvider,
|
||||
registry=_jsc_providers.value,
|
||||
)
|
||||
|
||||
|
||||
def register_preference(*providers: type[JsChallengeProvider]) -> typing.Callable[[Preference], Preference]:
|
||||
"""Register a preference for a JsChallengeProvider class."""
|
||||
return register_preference_generic(
|
||||
JsChallengeProvider,
|
||||
_jsc_preferences.value,
|
||||
*providers,
|
||||
)
|
||||
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
Preference = typing.Callable[[JsChallengeProvider, list[JsChallengeRequest]], int]
|
||||
__all__.append('Preference')
|
||||
@@ -6,6 +6,7 @@ import dataclasses
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import traceback
|
||||
import typing
|
||||
import urllib.parse
|
||||
from collections.abc import Iterable
|
||||
@@ -58,9 +59,9 @@ class YoutubeIEContentProviderLogger(IEContentProviderLogger):
|
||||
if self.log_level <= self.LogLevel.TRACE:
|
||||
self.__ie.write_debug(self._format_msg('TRACE: ' + message))
|
||||
|
||||
def debug(self, message: str):
|
||||
def debug(self, message: str, *, once=False):
|
||||
if self.log_level <= self.LogLevel.DEBUG:
|
||||
self.__ie.write_debug(self._format_msg(message))
|
||||
self.__ie.write_debug(self._format_msg(message), only_once=once)
|
||||
|
||||
def info(self, message: str):
|
||||
if self.log_level <= self.LogLevel.INFO:
|
||||
@@ -70,9 +71,11 @@ class YoutubeIEContentProviderLogger(IEContentProviderLogger):
|
||||
if self.log_level <= self.LogLevel.WARNING:
|
||||
self.__ie.report_warning(self._format_msg(message), only_once=once)
|
||||
|
||||
def error(self, message: str):
|
||||
def error(self, message: str, cause=None):
|
||||
if self.log_level <= self.LogLevel.ERROR:
|
||||
self.__ie._downloader.report_error(self._format_msg(message), is_error=False)
|
||||
self.__ie._downloader.report_error(
|
||||
self._format_msg(message), is_error=False,
|
||||
tb=''.join(traceback.format_exception(None, cause, cause.__traceback__)) if cause else None)
|
||||
|
||||
|
||||
class PoTokenCache:
|
||||
|
||||
@@ -36,7 +36,7 @@ class IEContentProviderLogger(abc.ABC):
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def debug(self, message: str):
|
||||
def debug(self, message: str, *, once=False):
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
@@ -48,7 +48,7 @@ class IEContentProviderLogger(abc.ABC):
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def error(self, message: str):
|
||||
def error(self, message: str, cause=None):
|
||||
pass
|
||||
|
||||
|
||||
@@ -90,7 +90,7 @@ class IEContentProvider(abc.ABC):
|
||||
@classproperty
|
||||
def PROVIDER_KEY(cls) -> str:
|
||||
assert hasattr(cls, '_PROVIDER_KEY_SUFFIX'), 'Content Provider implementation must define a suffix for the provider key'
|
||||
assert cls.__name__.endswith(cls._PROVIDER_KEY_SUFFIX), f'PoTokenProvider class names must end with "{cls._PROVIDER_KEY_SUFFIX}"'
|
||||
assert cls.__name__.endswith(cls._PROVIDER_KEY_SUFFIX), f'Class name must end with "{cls._PROVIDER_KEY_SUFFIX}"'
|
||||
return cls.__name__[:-len(cls._PROVIDER_KEY_SUFFIX)]
|
||||
|
||||
@abc.abstractmethod
|
||||
@@ -114,10 +114,7 @@ class IEContentProvider(abc.ABC):
|
||||
@param default The default value to return when the key is not present (default: [])
|
||||
@param casesense When false, the values are converted to lower case
|
||||
"""
|
||||
val = traverse_obj(self.settings, key)
|
||||
if val is None:
|
||||
return [] if default is NO_DEFAULT else default
|
||||
return list(val) if casesense else [x.lower() for x in val]
|
||||
return configuration_arg(self.settings, key, default=default, casesense=casesense)
|
||||
|
||||
|
||||
class BuiltinIEContentProvider(IEContentProvider, abc.ABC):
|
||||
@@ -125,6 +122,20 @@ class BuiltinIEContentProvider(IEContentProvider, abc.ABC):
|
||||
BUG_REPORT_MESSAGE = bug_reports_message(before='')
|
||||
|
||||
|
||||
def configuration_arg(config, key, default=NO_DEFAULT, *, casesense=False):
|
||||
"""
|
||||
@returns A list of values for the setting given by "key"
|
||||
or "default" if no such key is present
|
||||
@param config The configuration dictionary
|
||||
@param default The default value to return when the key is not present (default: [])
|
||||
@param casesense When false, the values are converted to lower case
|
||||
"""
|
||||
val = traverse_obj(config, key)
|
||||
if val is None:
|
||||
return [] if default is NO_DEFAULT else default
|
||||
return list(val) if casesense else [x.lower() for x in val]
|
||||
|
||||
|
||||
def register_provider_generic(
|
||||
provider,
|
||||
base_class,
|
||||
|
||||
@@ -127,6 +127,7 @@ class ZDFBaseIE(InfoExtractor):
|
||||
**parse_codecs(quality.get('mimeCodec')),
|
||||
'height': height,
|
||||
'width': width,
|
||||
'filesize': int_or_none(variant.get('filesize')),
|
||||
'format_id': join_nonempty('http', stream.get('type')),
|
||||
'tbr': int_or_none(self._search_regex(r'_(\d+)k_', format_url, 'tbr', default=None)),
|
||||
}]
|
||||
@@ -193,8 +194,10 @@ class ZDFBaseIE(InfoExtractor):
|
||||
class ZDFIE(ZDFBaseIE):
|
||||
_VALID_URL = [
|
||||
r'https?://(?:www\.)?zdf\.de/(?:video|play)/(?:[^/?#]+/)*(?P<id>[^/?#]+)',
|
||||
# /nachrichten/ sub-site URLs and legacy redirects from before the redesign in 2025-03
|
||||
# Legacy redirects from before the redesign in 2025-03 or from before sister sites moved to their own domains
|
||||
r'https?://(?:www\.)?zdf\.de/(?:[^/?#]+/)*(?P<id>[^/?#]+)\.html',
|
||||
# Sister sites
|
||||
r'https?://(?:www\.)?(?:zdfheute|logo)\.de/(?:[^/?#]+/)*(?P<id>[^/?#]+)\.html',
|
||||
]
|
||||
IE_NAME = 'zdf'
|
||||
_TESTS = [{
|
||||
@@ -206,7 +209,7 @@ class ZDFIE(ZDFBaseIE):
|
||||
'title': 'Sylt - Deutschlands edles Nordlicht',
|
||||
'description': 'md5:35407b810c2e1e33efbe15ef6e4c06c3',
|
||||
'duration': 810.0,
|
||||
'thumbnail': 'https://www.zdf.de/assets/sylt-118~original?cb=1613992485011',
|
||||
'thumbnail': r're:https://www\.zdf\.de/assets/sylt-118~original\?cb=\d+',
|
||||
'series': 'Sylt - Deutschlands edles Nordlicht',
|
||||
'series_id': 'sylt---deutschlands-edles-nordlicht-movie-100',
|
||||
'timestamp': 1612462500,
|
||||
@@ -222,7 +225,7 @@ class ZDFIE(ZDFBaseIE):
|
||||
'title': 'Sylt - Deutschlands edles Nordlicht',
|
||||
'description': 'md5:35407b810c2e1e33efbe15ef6e4c06c3',
|
||||
'duration': 810.0,
|
||||
'thumbnail': 'https://www.zdf.de/assets/sylt-118~original?cb=1613992485011',
|
||||
'thumbnail': r're:https://www\.zdf\.de/assets/sylt-118~original\?cb=\d+',
|
||||
'series': 'Sylt - Deutschlands edles Nordlicht',
|
||||
'series_id': 'sylt---deutschlands-edles-nordlicht-movie-100',
|
||||
'timestamp': 1612462500,
|
||||
@@ -239,7 +242,7 @@ class ZDFIE(ZDFBaseIE):
|
||||
'title': 'Sylt - Deutschlands edles Nordlicht',
|
||||
'description': 'md5:35407b810c2e1e33efbe15ef6e4c06c3',
|
||||
'duration': 810.0,
|
||||
'thumbnail': 'https://www.zdf.de/assets/sylt-118~original?cb=1613992485011',
|
||||
'thumbnail': r're:https://www\.zdf\.de/assets/sylt-118~original\?cb=\d+',
|
||||
'series': 'Sylt - Deutschlands edles Nordlicht',
|
||||
'series_id': 'sylt---deutschlands-edles-nordlicht-movie-100',
|
||||
'timestamp': 1612462500,
|
||||
@@ -249,67 +252,66 @@ class ZDFIE(ZDFBaseIE):
|
||||
'params': {'skip_download': True},
|
||||
}, {
|
||||
# Video belongs to a playlist, video URL
|
||||
'url': 'https://www.zdf.de/video/dokus/die-magie-der-farben-116/die-magie-der-farben-von-koenigspurpur-und-jeansblau-100',
|
||||
'md5': '1eda17eb40a9ead3046326e10b9c5973',
|
||||
# Also: video mirrored from ARD Mediathek
|
||||
'url': 'https://www.zdf.de/video/dokus/collection-index-page-ard-collection-ard-dxjuomfyzdpzag93ojy2mzhhmmq3mzk2ztq4nda-132/page-video-ard-gelb-vom-hellen-glanz-zu-finsteren-abgruenden-102',
|
||||
'md5': '84980c1a0148da6cd94de58333d7e1ee',
|
||||
'info_dict': {
|
||||
'id': 'die-magie-der-farben-von-koenigspurpur-und-jeansblau-100',
|
||||
'id': 'page-video-ard-gelb-vom-hellen-glanz-zu-finsteren-abgruenden-102',
|
||||
'ext': 'mp4',
|
||||
'title': 'Von Königspurpur bis Jeansblau',
|
||||
'description': 'md5:a89da10c928c6235401066b60a6d5c1a',
|
||||
'duration': 2615.0,
|
||||
'thumbnail': 'https://www.zdf.de/assets/koenigspurpur-bis-jeansblau-100~original?cb=1741857765971',
|
||||
'title': 'Gelb: Vom hellen Glanz zu finsteren Abgründen',
|
||||
'description': 'md5:9aad4806b4c8ea152ab21e70c9d516be',
|
||||
'duration': 895.0,
|
||||
'thumbnail': r're:https://www\.zdf\.de/assets/image-ard-gelb-vom-hellen-glanz-zu-finsteren-abgruenden-102~original\?cb=\d+',
|
||||
'series': 'Die Magie der Farben',
|
||||
'series_id': 'die-magie-der-farben-116',
|
||||
'season': 'Season 1',
|
||||
'season_number': 1,
|
||||
'episode': 'Episode 2',
|
||||
'episode_number': 2,
|
||||
'timestamp': 1445797800,
|
||||
'upload_date': '20151025',
|
||||
'_old_archive_ids': ['zdf 151025_magie_farben2_tex'],
|
||||
'series_id': 'collection-index-page-ard-collection-ard-dxjuomfyzdpzag93ojy2mzhhmmq3mzk2ztq4nda-132',
|
||||
'season': 'Season 2023',
|
||||
'season_number': 2023,
|
||||
'episode': 'Episode 5',
|
||||
'episode_number': 5,
|
||||
'timestamp': 1690902120,
|
||||
'upload_date': '20230801',
|
||||
'_old_archive_ids': ['zdf video_ard_dXJuOmFyZDpwdWJsaWNhdGlvbjo0YTYyOTJjM2Q0ZThlNmY1'],
|
||||
},
|
||||
}, {
|
||||
# Video belongs to a playlist, play URL
|
||||
'url': 'https://www.zdf.de/play/dokus/die-magie-der-farben-116/die-magie-der-farben-von-koenigspurpur-und-jeansblau-100',
|
||||
'md5': '1eda17eb40a9ead3046326e10b9c5973',
|
||||
'url': 'https://www.zdf.de/play/dokus/collection-index-page-ard-collection-ard-dxjuomfyzdpzag93ojy2mzhhmmq3mzk2ztq4nda-132/page-video-ard-gelb-vom-hellen-glanz-zu-finsteren-abgruenden-102',
|
||||
'info_dict': {
|
||||
'id': 'die-magie-der-farben-von-koenigspurpur-und-jeansblau-100',
|
||||
'id': 'page-video-ard-gelb-vom-hellen-glanz-zu-finsteren-abgruenden-102',
|
||||
'ext': 'mp4',
|
||||
'title': 'Von Königspurpur bis Jeansblau',
|
||||
'description': 'md5:a89da10c928c6235401066b60a6d5c1a',
|
||||
'duration': 2615.0,
|
||||
'thumbnail': 'https://www.zdf.de/assets/koenigspurpur-bis-jeansblau-100~original?cb=1741857765971',
|
||||
'title': 'Gelb: Vom hellen Glanz zu finsteren Abgründen',
|
||||
'description': 'md5:9aad4806b4c8ea152ab21e70c9d516be',
|
||||
'duration': 895.0,
|
||||
'thumbnail': r're:https://www\.zdf\.de/assets/image-ard-gelb-vom-hellen-glanz-zu-finsteren-abgruenden-102~original\?cb=\d+',
|
||||
'series': 'Die Magie der Farben',
|
||||
'series_id': 'die-magie-der-farben-116',
|
||||
'season': 'Season 1',
|
||||
'season_number': 1,
|
||||
'episode': 'Episode 2',
|
||||
'episode_number': 2,
|
||||
'timestamp': 1445797800,
|
||||
'upload_date': '20151025',
|
||||
'_old_archive_ids': ['zdf 151025_magie_farben2_tex'],
|
||||
'series_id': 'collection-index-page-ard-collection-ard-dxjuomfyzdpzag93ojy2mzhhmmq3mzk2ztq4nda-132',
|
||||
'season': 'Season 2023',
|
||||
'season_number': 2023,
|
||||
'episode': 'Episode 5',
|
||||
'episode_number': 5,
|
||||
'timestamp': 1690902120,
|
||||
'upload_date': '20230801',
|
||||
'_old_archive_ids': ['zdf video_ard_dXJuOmFyZDpwdWJsaWNhdGlvbjo0YTYyOTJjM2Q0ZThlNmY1'],
|
||||
},
|
||||
'params': {'skip_download': True},
|
||||
}, {
|
||||
# Video belongs to a playlist, legacy URL before website redesign in 2025-03
|
||||
'url': 'https://www.zdf.de/dokumentation/terra-x/die-magie-der-farben-von-koenigspurpur-und-jeansblau-100.html',
|
||||
'md5': '1eda17eb40a9ead3046326e10b9c5973',
|
||||
'url': 'https://www.zdf.de/dokus/collection-index-page-ard-collection-ard-dxjuomfyzdpzag93ojy2mzhhmmq3mzk2ztq4nda-132/page-video-ard-gelb-vom-hellen-glanz-zu-finsteren-abgruenden-102.html',
|
||||
'info_dict': {
|
||||
'id': 'die-magie-der-farben-von-koenigspurpur-und-jeansblau-100',
|
||||
'id': 'page-video-ard-gelb-vom-hellen-glanz-zu-finsteren-abgruenden-102',
|
||||
'ext': 'mp4',
|
||||
'title': 'Von Königspurpur bis Jeansblau',
|
||||
'description': 'md5:a89da10c928c6235401066b60a6d5c1a',
|
||||
'duration': 2615.0,
|
||||
'thumbnail': 'https://www.zdf.de/assets/koenigspurpur-bis-jeansblau-100~original?cb=1741857765971',
|
||||
'title': 'Gelb: Vom hellen Glanz zu finsteren Abgründen',
|
||||
'description': 'md5:9aad4806b4c8ea152ab21e70c9d516be',
|
||||
'duration': 895.0,
|
||||
'thumbnail': r're:https://www\.zdf\.de/assets/image-ard-gelb-vom-hellen-glanz-zu-finsteren-abgruenden-102~original\?cb=\d+',
|
||||
'series': 'Die Magie der Farben',
|
||||
'series_id': 'die-magie-der-farben-116',
|
||||
'season': 'Season 1',
|
||||
'season_number': 1,
|
||||
'episode': 'Episode 2',
|
||||
'episode_number': 2,
|
||||
'timestamp': 1445797800,
|
||||
'upload_date': '20151025',
|
||||
'_old_archive_ids': ['zdf 151025_magie_farben2_tex'],
|
||||
'series_id': 'collection-index-page-ard-collection-ard-dxjuomfyzdpzag93ojy2mzhhmmq3mzk2ztq4nda-132',
|
||||
'season': 'Season 2023',
|
||||
'season_number': 2023,
|
||||
'episode': 'Episode 5',
|
||||
'episode_number': 5,
|
||||
'timestamp': 1690902120,
|
||||
'upload_date': '20230801',
|
||||
'_old_archive_ids': ['zdf video_ard_dXJuOmFyZDpwdWJsaWNhdGlvbjo0YTYyOTJjM2Q0ZThlNmY1'],
|
||||
},
|
||||
'params': {'skip_download': True},
|
||||
}, {
|
||||
@@ -337,7 +339,50 @@ class ZDFIE(ZDFBaseIE):
|
||||
'_old_archive_ids': ['zdf 211219_sendung_hjo'],
|
||||
},
|
||||
}, {
|
||||
# Video that requires fallback extraction
|
||||
# FUNK video (hosted on a different CDN, has atypical PTMD and HLS files)
|
||||
'url': 'https://www.zdf.de/video/serien/funk-collection-funk-11790-1596/funk-alles-ist-verzaubert-102',
|
||||
'md5': '57af4423db0455a3975d2dc4578536bc',
|
||||
'info_dict': {
|
||||
'id': 'funk-alles-ist-verzaubert-102',
|
||||
'ext': 'mp4',
|
||||
'title': 'Alles ist verzaubert',
|
||||
'description': 'Die Neue an der Schule verdreht Ismail den Kopf.',
|
||||
'duration': 1278.0,
|
||||
'thumbnail': r're:https://www\.zdf\.de/assets/teaser-funk-alles-ist-verzaubert-102~original\?cb=\d+',
|
||||
'series': 'DRUCK',
|
||||
'series_id': 'funk-collection-funk-11790-1596',
|
||||
'season': 'Season 2021',
|
||||
'season_number': 2021,
|
||||
'episode': 'Episode 50',
|
||||
'episode_number': 50,
|
||||
'timestamp': 1635520560,
|
||||
'upload_date': '20211029',
|
||||
'_old_archive_ids': ['zdf video_funk_1770473'],
|
||||
},
|
||||
}, {
|
||||
# zdfheute video, also available on zdf.de
|
||||
'url': 'https://www.zdfheute.de/video/heute-journal/heute-journal-vom-19-dezember-2025-100.html',
|
||||
'md5': '47af8c2cfa30abf74501170f62754c63',
|
||||
'info_dict': {
|
||||
'id': 'heute-journal-vom-19-dezember-2025-100',
|
||||
'ext': 'mp4',
|
||||
'title': 'heute journal vom 19. Dezember 2025',
|
||||
'description': 'md5:fd0dfbce0783486db839ff9140a8074b',
|
||||
'duration': 1780.0,
|
||||
'thumbnail': 'https://epg-image.zdf.de/fotobase-webdelivery/images/273e5545-16e7-4ca3-898e-52fe9e06d964?layout=2400x1350',
|
||||
'chapters': 'count:10',
|
||||
'series': 'heute journal',
|
||||
'series_id': 'heute-journal-104',
|
||||
'season': 'Season 2025',
|
||||
'season_number': 2025,
|
||||
'episode': 'Episode 365',
|
||||
'episode_number': 365,
|
||||
'timestamp': 1766178000,
|
||||
'upload_date': '20251219',
|
||||
'_old_archive_ids': ['zdf 251219_2200_sendung_hjo'],
|
||||
},
|
||||
}, {
|
||||
# zdfheute video, not available on zdf.de (uses the fallback extraction path)
|
||||
'url': 'https://www.zdf.de/nachrichten/politik/deutschland/koalitionsverhandlungen-spd-cdu-csu-dobrindt-100.html',
|
||||
'md5': 'c3a78514dd993a5781aa3afe50db51e2',
|
||||
'info_dict': {
|
||||
@@ -346,71 +391,50 @@ class ZDFIE(ZDFBaseIE):
|
||||
'title': 'Dobrindt schließt Steuererhöhungen aus',
|
||||
'description': 'md5:9a117646d7b8df6bc902eb543a9c9023',
|
||||
'duration': 325,
|
||||
'thumbnail': 'https://www.zdfheute.de/assets/dobrindt-csu-berlin-direkt-100~1920x1080?cb=1743357653736',
|
||||
'thumbnail': r're:https://www\.zdfheute\.de/assets/dobrindt-csu-berlin-direkt-100~1920x1080\?cb=\d+',
|
||||
'timestamp': 1743374520,
|
||||
'upload_date': '20250330',
|
||||
'_old_archive_ids': ['zdf 250330_clip_2_bdi'],
|
||||
},
|
||||
}, {
|
||||
# FUNK video (hosted on a different CDN, has atypical PTMD and HLS files)
|
||||
'url': 'https://www.zdf.de/funk/druck-11790/funk-alles-ist-verzaubert-102.html',
|
||||
'md5': '57af4423db0455a3975d2dc4578536bc',
|
||||
# logo! video, also available on zdf.de
|
||||
'url': 'https://www.logo.de/logo-vom-freitag-19-dezember-2025-102.html',
|
||||
'md5': 'cfb1a0988b1249f052a437a55851134b',
|
||||
'info_dict': {
|
||||
'id': 'funk-alles-ist-verzaubert-102',
|
||||
'id': 'logo-vom-freitag-19-dezember-2025-102',
|
||||
'ext': 'mp4',
|
||||
'title': 'Alles ist verzaubert',
|
||||
'description': 'Die Neue an der Schule verdreht Ismail den Kopf.',
|
||||
'duration': 1278.0,
|
||||
'thumbnail': 'https://www.zdf.de/assets/teaser-funk-alles-ist-verzaubert-102~original?cb=1663848412907',
|
||||
'series': 'DRUCK',
|
||||
'series_id': 'funk-collection-funk-11790-1590',
|
||||
'season': 'Season 7',
|
||||
'season_number': 7,
|
||||
'episode': 'Episode 1',
|
||||
'episode_number': 1,
|
||||
'timestamp': 1635520560,
|
||||
'upload_date': '20211029',
|
||||
'_old_archive_ids': ['zdf video_funk_1770473'],
|
||||
'title': 'logo! vom Freitag, 19. Dezember 2025',
|
||||
'description': 'md5:971428cb563e924c153580f23870c613',
|
||||
'duration': 490.0,
|
||||
'thumbnail': r're:https://www\.zdf\.de/assets/iran-rote-erde-sendung-19-dezember-2025-100~original\?cb=\d+',
|
||||
'chapters': 'count:7',
|
||||
'series': 'logo!',
|
||||
'series_id': 'logo-154',
|
||||
'season': 'Season 2025',
|
||||
'season_number': 2025,
|
||||
'episode': 'Episode 382',
|
||||
'episode_number': 382,
|
||||
'timestamp': 1766168700,
|
||||
'upload_date': '20251219',
|
||||
'_old_archive_ids': ['zdf 251219_1925_sendung_log'],
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.zdf.de/serien/soko-stuttgart/das-geld-anderer-leute-100.html',
|
||||
# logo! video, not available on zdf.de (uses the fallback extraction path)
|
||||
'url': 'https://www.logo.de/kinderreporter-vivaan-trifft-alina-grijseels-100.html',
|
||||
'md5': '094cea026babb67aa25fd0108400bc12',
|
||||
'info_dict': {
|
||||
'id': 'das-geld-anderer-leute-100',
|
||||
'id': 'kinderreporter-vivaan-trifft-alina-grijseels-100',
|
||||
'ext': 'mp4',
|
||||
'title': 'Das Geld anderer Leute',
|
||||
'description': 'md5:cb6f660850dc5eb7d1ab776ea094959d',
|
||||
'duration': 2581.0,
|
||||
'thumbnail': 'https://epg-image.zdf.de/fotobase-webdelivery/images/e2d7e55a-09f0-424e-ac73-6cac4dd65f35?layout=1920x1080',
|
||||
'series': 'SOKO Stuttgart',
|
||||
'series_id': 'soko-stuttgart-104',
|
||||
'season': 'Season 11',
|
||||
'season_number': 11,
|
||||
'episode': 'Episode 10',
|
||||
'episode_number': 10,
|
||||
'timestamp': 1728983700,
|
||||
'upload_date': '20241015',
|
||||
'_old_archive_ids': ['zdf 191205_1800_sendung_sok8'],
|
||||
'title': 'Vivaan trifft Handballerin Alina Grijseels',
|
||||
'description': 'md5:9572e7f4340dda823ea4091a76624da6',
|
||||
'duration': 166.0,
|
||||
'thumbnail': r're:https://www\.zdf\.de/assets/vivaan-alina-grijseels-100~original\?cb=\d+',
|
||||
'series': 'logo!',
|
||||
'series_id': 'logo-154',
|
||||
'timestamp': 1766236320,
|
||||
'upload_date': '20251220',
|
||||
'_old_archive_ids': ['zdf 251219_kr_alina_grijseels_neu_log'],
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.zdf.de/serien/northern-lights/begegnung-auf-der-bruecke-100.html',
|
||||
'info_dict': {
|
||||
'id': 'begegnung-auf-der-bruecke-100',
|
||||
'ext': 'webm',
|
||||
'title': 'Begegnung auf der Brücke',
|
||||
'description': 'md5:e53a555da87447f7f1207f10353f8e45',
|
||||
'duration': 3083.0,
|
||||
'thumbnail': 'https://epg-image.zdf.de/fotobase-webdelivery/images/c5ff1d1f-f5c8-4468-86ac-1b2f1dbecc76?layout=1920x1080',
|
||||
'series': 'Northern Lights',
|
||||
'series_id': 'northern-lights-100',
|
||||
'season': 'Season 1',
|
||||
'season_number': 1,
|
||||
'episode': 'Episode 1',
|
||||
'episode_number': 1,
|
||||
'timestamp': 1738546500,
|
||||
'upload_date': '20250203',
|
||||
'_old_archive_ids': ['zdf 240319_2310_sendung_not'],
|
||||
},
|
||||
'params': {'skip_download': 'geo-restricted http format'},
|
||||
}, {
|
||||
# Same as https://www.phoenix.de/sendungen/ereignisse/corona-nachgehakt/wohin-fuehrt-der-protest-in-der-pandemie-a-2050630.html
|
||||
'url': 'https://www.zdf.de/politik/phoenix-sendungen/wohin-fuehrt-der-protest-in-der-pandemie-100.html',
|
||||
@@ -617,7 +641,7 @@ class ZDFChannelIE(ZDFBaseIE):
|
||||
'title': 'Sylt - Deutschlands edles Nordlicht',
|
||||
'description': 'md5:35407b810c2e1e33efbe15ef6e4c06c3',
|
||||
'duration': 810.0,
|
||||
'thumbnail': 'https://www.zdf.de/assets/sylt-118~original?cb=1613992485011',
|
||||
'thumbnail': r're:https://www\.zdf\.de/assets/sylt-118~original\?cb=\d+',
|
||||
'series': 'Sylt - Deutschlands edles Nordlicht',
|
||||
'series_id': 'sylt---deutschlands-edles-nordlicht-movie-100',
|
||||
'timestamp': 1612462500,
|
||||
@@ -634,38 +658,24 @@ class ZDFChannelIE(ZDFBaseIE):
|
||||
},
|
||||
'playlist_mincount': 2,
|
||||
}, {
|
||||
'url': 'https://www.zdf.de/serien/taunuskrimi/',
|
||||
'info_dict': {
|
||||
'id': 'taunuskrimi-100',
|
||||
'title': 'Taunuskrimi',
|
||||
'description': 'md5:ee7204e9c625c3b611d1274f9d0e3070',
|
||||
},
|
||||
'playlist_mincount': 8,
|
||||
}, {
|
||||
'url': 'https://www.zdf.de/serien/taunuskrimi/?staffel=1',
|
||||
'info_dict': {
|
||||
'id': 'taunuskrimi-100-s1',
|
||||
'title': 'Taunuskrimi - Season 1',
|
||||
'description': 'md5:ee7204e9c625c3b611d1274f9d0e3070',
|
||||
},
|
||||
'playlist_count': 7,
|
||||
}, {
|
||||
# All seasons of playlist
|
||||
'url': 'https://www.zdf.de/magazine/heute-journal-104',
|
||||
'info_dict': {
|
||||
'id': 'heute-journal-104',
|
||||
'title': 'heute journal',
|
||||
'description': 'md5:6edad39189abf8431795d3d6d7f986b3',
|
||||
},
|
||||
'playlist_mincount': 500,
|
||||
'playlist_mincount': 366,
|
||||
}, {
|
||||
'url': 'https://www.zdf.de/magazine/heute-journal-104?staffel=2024',
|
||||
# Only selected season
|
||||
'url': 'https://www.zdf.de/magazine/heute-journal-104?staffel=2025',
|
||||
'info_dict': {
|
||||
'id': 'heute-journal-104-s2024',
|
||||
'title': 'heute journal - Season 2024',
|
||||
'id': 'heute-journal-104-s2025',
|
||||
'title': 'heute journal - Season 2025',
|
||||
'description': 'md5:6edad39189abf8431795d3d6d7f986b3',
|
||||
},
|
||||
'playlist_count': 242,
|
||||
'skip': 'Video count changes daily, needs support for playlist_maxcount',
|
||||
'playlist_mincount': 1,
|
||||
'playlist_maxcount': 365,
|
||||
}]
|
||||
|
||||
_PAGE_SIZE = 24
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user