itdominator/SolarFM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

restructured manifest and plugins loading; updated plugins

Browse Source
This commit is contained in:
itdominator
2025-12-29 22:50:05 -06:00
parent c74f97aca7
commit 21120cd61e
324 changed files with 18088 additions and 15974 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/archiver/manifest.json
+7 -9
View File
@@ -1,12 +1,10 @@
{ {
"manifest": { "name": "Archiver",
"name": "Archiver", "author": "ITDominator",
"author": "ITDominator", "version": "0.0.1",
"version": "0.0.1", "support": "",
"support": "", "requests": {
"requests": { "ui_target": "context_menu_plugins",
"ui_target": "context_menu_plugins", "pass_fm_events": true
"pass_fm_events": "true"
}
} }
} }
plugins/disk_usage/manifest.json
+7 -9
View File
@@ -1,12 +1,10 @@
{ {
"manifest": { "name": "Disk Usage",
"name": "Disk Usage", "author": "ITDominator",
"author": "ITDominator", "version": "0.0.1",
"version": "0.0.1", "support": "",
"support": "", "requests": {
"requests": { "ui_target": "context_menu_plugins",
"ui_target": "context_menu_plugins", "pass_fm_events": true
"pass_fm_events": "true"
}
} }
} }
plugins/favorites/manifest.json
+9 -11
View File
@@ -1,14 +1,12 @@
{ {
"manifest": { "name": "Favorites",
"name": "Favorites", "author": "ITDominator",
"author": "ITDominator", "version": "0.0.1",
"version": "0.0.1", "support": "",
"support": "", "requests": {
"requests": { "ui_target": "main_menu_bttn_box_bar",
"ui_target": "main_menu_bttn_box_bar", "pass_fm_events": true,
"pass_fm_events": "true", "pass_ui_objects": ["path_entry"],
"pass_ui_objects": ["path_entry"], "bind_keys": ["Favorites||show_favorites_menu:<Control>f"]
"bind_keys": ["Favorites||show_favorites_menu:<Control>f"]
}
} }
} }
plugins/file_properties/manifest.json
+7 -9
View File
@@ -1,12 +1,10 @@
{ {
"manifest": { "name": "Properties",
"name": "Properties", "author": "ITDominator",
"author": "ITDominator", "version": "0.0.1",
"version": "0.0.1", "support": "",
"support": "", "requests": {
"requests": { "ui_target": "context_menu",
"ui_target": "context_menu", "pass_fm_events": true
"pass_fm_events": "true"
}
} }
} }
plugins/git_clone/manifest.json
+7 -9
View File
@@ -1,12 +1,10 @@
{ {
"manifest": { "name": "Git Clone",
"name": "Git Clone", "author": "ITDominator",
"author": "ITDominator", "version": "0.0.1",
"version": "0.0.1", "support": "",
"support": "", "requests": {
"requests": { "ui_target": "plugin_control_list",
"ui_target": "plugin_control_list", "pass_fm_events": true
"pass_fm_events": "true"
}
} }
} }
plugins/movie_tv_info/manifest.json
+7 -9
View File
@@ -1,12 +1,10 @@
{ {
"manifest": { "name": "Movie/TV Info",
"name": "Movie/TV Info", "author": "ITDominator",
"author": "ITDominator", "version": "0.0.1",
"version": "0.0.1", "support": "",
"support": "", "requests": {
"requests": { "ui_target": "context_menu_plugins",
"ui_target": "context_menu_plugins", "pass_fm_events": true
"pass_fm_events": "true"
}
} }
} }
plugins/py_run/manifest.json
+8 -10
View File
@@ -1,13 +1,11 @@
{ {
"manifest": { "name": "PyRun",
"name": "PyRun", "author": "ITDominator",
"author": "ITDominator", "version": "0.0.1",
"version": "0.0.1", "support": "",
"support": "", "requests": {
"requests": { "ui_target": "plugin_control_list",
"ui_target": "plugin_control_list", "pass_fm_events": true,
"pass_fm_events": "true", "bind_keys": ["PyRun||send_message:<Shift><Control>r"]
"bind_keys": ["PyRun||send_message:<Shift><Control>r"]
}
} }
} }
plugins/searcher/manifest.json
+8 -10
View File
@@ -1,13 +1,11 @@
{ {
"manifest": { "name": "Search",
"name": "Search", "author": "ITDominator",
"author": "ITDominator", "version": "0.0.1",
"version": "0.0.1", "support": "",
"support": "", "requests": {
"requests": { "ui_target": "context_menu",
"ui_target": "context_menu", "pass_fm_events": true,
"pass_fm_events": "true", "bind_keys": ["Search||show_search_page:<Control>s"]
"bind_keys": ["Search||show_search_page:<Control>s"]
}
} }
} }
plugins/searcher/mixins/file_search_mixin.py
+28 -28
View File
@@ -18,22 +18,22 @@ from ..widgets.file_preview_widget import FilePreviewWidget
# NOTE: Threads WILL NOT die with parent's destruction. # NOTE: Threads WILL NOT die with parent's destruction.
def threaded(fn): # def threaded(fn):
def wrapper(*args, **kwargs): # def wrapper(*args, **kwargs):
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start() # threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
#
return wrapper # return wrapper
#
# NOTE: Threads WILL die with parent's destruction. # NOTE: Threads WILL die with parent's destruction.
def daemon_threaded(fn): # def daemon_threaded(fn):
def wrapper(*args, **kwargs): # def wrapper(*args, **kwargs):
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start() # threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
#
return wrapper # return wrapper
class FileSearchMixin: class FileSearchMixin:
def _run_find_file_query(self, widget=None, eve=None): def _run_find_file_query(self, widget = None, eve = None):
self._queue_search = True self._queue_search = True
if not self._search_watcher_running: if not self._search_watcher_running:
@@ -52,22 +52,22 @@ class FileSearchMixin:
# Need to implement this over the threaded stuffs.... # Need to implement this over the threaded stuffs....
#
def cancel_timer(self): # def cancel_timer(self):
if self.timer: # if self.timer:
self.timer.cancel() # self.timer.cancel()
GLib.idle_remove_by_data(None) # GLib.idle_remove_by_data(None)
#
def delay_search_Glib(self): # def delay_search_glib(self, query):
GLib.idle_add(self._do_highlight) # GLib.idle_add(self._exec_find_file_query, *(query,))
#
def delay_search(self): # def delay_search(self):
wait_time = self.search_time / len(self.search_text) # wait_time = self.search_time / len(self.search_text)
wait_time = max(wait_time, 0.05) # wait_time = max(wait_time, 0.05)
#
self.timer = threading.Timer(wait_time, self.delay_search_Glib) # self.timer = threading.Timer(wait_time, self.delay_search_glib, *(query,))
self.timer.daemon = True # self.timer.daemon = True
self.timer.start() # self.timer.start()
plugins/template/manifest.json
+10 -12
View File
@@ -1,14 +1,12 @@
{ {
"manifest": { "name": "Example Plugin",
"name": "Example Plugin", "author": "John Doe",
"author": "John Doe", "version": "0.0.1",
"version": "0.0.1", "support": "",
"support": "", "requests": {
"requests": { "ui_target": "plugin_control_list",
"ui_target": "plugin_control_list", "pass_fm_events": true,
"pass_fm_events": "true", "bind_keys": ["Example Plugin||send_message:<Control>f"]
"bind_keys": ["Example Plugin||send_message:<Control>f"] },
}, "pre_launch": false
"pre_launch": "false"
}
} }
plugins/thumbnailer/manifest.json
+7 -9
View File
@@ -1,12 +1,10 @@
{ {
"manifest": { "name": "Thumbnailer",
"name": "Thumbnailer", "author": "ITDominator",
"author": "ITDominator", "version": "0.0.1",
"version": "0.0.1", "support": "",
"support": "", "pre_launch": true,
"requests": { "requests": {
"pass_fm_events": "true" "pass_fm_events": true
},
"pre_launch": "true"
} }
} }
plugins/translate/manifest.json
+1 -3
View File
@@ -1,12 +1,10 @@
{ {
"manifest": {
"name": "Translate", "name": "Translate",
"author": "ITDominator", "author": "ITDominator",
"version": "0.0.1", "version": "0.0.1",
"support": "", "support": "",
"requests": { "requests": {
"ui_target": "plugin_control_list", "ui_target": "plugin_control_list",
"pass_fm_events": "true" "pass_fm_events": true
} }
}
} }
plugins/trasher/manifest.json
+11 -13
View File
@@ -1,16 +1,14 @@
{ {
"manifest": { "name": "Trasher",
"name": "Trasher", "author": "ITDominator",
"author": "ITDominator", "version": "0.0.1",
"version": "0.0.1", "support": "",
"support": "", "requests": {
"requests": { "ui_target": "context_menu",
"ui_target": "context_menu", "pass_fm_events": true,
"pass_fm_events": "true", "bind_keys": [
"bind_keys": [ "Trasher||delete_files:Delete",
"Trasher||delete_files:Delete", "Trasher||trash_files:<Control>d"
"Trasher||trash_files:<Control>d" ]
]
}
} }
} }
plugins/vod_thumbnailer/manifest.json
+7 -9
View File
@@ -1,12 +1,10 @@
{ {
"manifest": { "name": "VOD Thumbnailer",
"name": "VOD Thumbnailer", "author": "ITDominator",
"author": "ITDominator", "version": "0.0.1",
"version": "0.0.1", "support": "",
"support": "", "requests": {
"requests": { "ui_target": "context_menu_plugins",
"ui_target": "context_menu_plugins", "pass_fm_events": true
"pass_fm_events": "true"
}
} }
} }
plugins/youtube_download/download.sh
+1 -1
View File
@@ -28,7 +28,7 @@ function main() {
# Note: Download video # Note: Download video
python "${_SPATH}/yt_dlp/__main__.py" \ python "${_SPATH}/yt_dlp/__main__.py" \
-f "bestvideo[height<=1080][ext=mp4][vcodec^=avc]+bestaudio[ext=m4a]/best[ext=mp4]/best" \ -f "bestvideo[height<=1080][ext=mp4][vcodec^=av]+bestaudio[ext=m4a]/best[ext=mp4]/best" \
--cookies "${_SPATH}/../../cookies.txt" --write-sub --embed-sub --sub-langs en \ --cookies "${_SPATH}/../../cookies.txt" --write-sub --embed-sub --sub-langs en \
-o "${_STARGET}/%(title)s.%(ext)s" "${LINK}" -o "${_STARGET}/%(title)s.%(ext)s" "${LINK}"
plugins/youtube_download/manifest.json
+7 -9
View File
@@ -1,12 +1,10 @@
{ {
"manifest": { "name": "Youtube Download",
"name": "Youtube Download", "author": "ITDominator",
"author": "ITDominator", "version": "0.0.1",
"version": "0.0.1", "support": "",
"support": "", "requests": {
"requests": { "ui_target": "plugin_control_list",
"ui_target": "plugin_control_list", "pass_fm_events": true
"pass_fm_events": "true"
}
} }
} }
plugins/youtube_download/yt_dlp/YoutubeDL.py
+77 -74
View File
@@ -36,6 +36,7 @@ from .extractor.openload import PhantomJSwrapper
from .globals import ( from .globals import (
IN_CLI, IN_CLI,
LAZY_EXTRACTORS, LAZY_EXTRACTORS,
WINDOWS_VT_MODE,
plugin_ies, plugin_ies,
plugin_ies_overrides, plugin_ies_overrides,
plugin_pps, plugin_pps,
@@ -52,7 +53,7 @@ from .networking.exceptions import (
SSLError, SSLError,
network_exceptions, network_exceptions,
) )
from .networking.impersonate import ImpersonateRequestHandler from .networking.impersonate import ImpersonateRequestHandler, ImpersonateTarget
from .plugins import directories as plugin_directories, load_all_plugins from .plugins import directories as plugin_directories, load_all_plugins
from .postprocessor import ( from .postprocessor import (
EmbedThumbnailPP, EmbedThumbnailPP,
@@ -72,6 +73,7 @@ from .postprocessor.ffmpeg import resolve_mapping as resolve_recode_mapping
from .update import ( from .update import (
REPOSITORY, REPOSITORY,
_get_system_deprecation, _get_system_deprecation,
_get_outdated_warning,
_make_label, _make_label,
current_git_head, current_git_head,
detect_variant, detect_variant,
@@ -302,7 +304,6 @@ class YoutubeDL:
clean_infojson: Remove internal metadata from the infojson clean_infojson: Remove internal metadata from the infojson
getcomments: Extract video comments. This will not be written to disk getcomments: Extract video comments. This will not be written to disk
unless writeinfojson is also given unless writeinfojson is also given
writeannotations: Write the video annotations to a .annotations.xml file
writethumbnail: Write the thumbnail image to a file writethumbnail: Write the thumbnail image to a file
allow_playlist_files: Whether to write playlists' description, infojson etc allow_playlist_files: Whether to write playlists' description, infojson etc
also to disk when using the 'write*' options also to disk when using the 'write*' options
@@ -482,7 +483,8 @@ class YoutubeDL:
The following options do not work when used through the API: The following options do not work when used through the API:
filename, abort-on-error, multistreams, no-live-chat, filename, abort-on-error, multistreams, no-live-chat,
format-sort, no-clean-infojson, no-playlist-metafiles, format-sort, no-clean-infojson, no-playlist-metafiles,
no-keep-subs, no-attach-info-json, allow-unsafe-ext, prefer-vp9-sort. no-keep-subs, no-attach-info-json, allow-unsafe-ext, prefer-vp9-sort,
mtime-by-default.
Refer __init__.py for their implementation Refer __init__.py for their implementation
progress_template: Dictionary of templates for progress outputs. progress_template: Dictionary of templates for progress outputs.
Allowed keys are 'download', 'postprocess', Allowed keys are 'download', 'postprocess',
@@ -490,7 +492,7 @@ class YoutubeDL:
The template is mapped on a dictionary with keys 'progress' and 'info' The template is mapped on a dictionary with keys 'progress' and 'info'
retry_sleep_functions: Dictionary of functions that takes the number of attempts retry_sleep_functions: Dictionary of functions that takes the number of attempts
as argument and returns the time to sleep in seconds. as argument and returns the time to sleep in seconds.
Allowed keys are 'http', 'fragment', 'file_access' Allowed keys are 'http', 'fragment', 'file_access', 'extractor'
download_ranges: A callback function that gets called for every video with download_ranges: A callback function that gets called for every video with
the signature (info_dict, ydl) -> Iterable[Section]. the signature (info_dict, ydl) -> Iterable[Section].
Only the returned sections will be downloaded. Only the returned sections will be downloaded.
@@ -502,16 +504,17 @@ class YoutubeDL:
force_keyframes_at_cuts: Re-encode the video when downloading ranges to get precise cuts force_keyframes_at_cuts: Re-encode the video when downloading ranges to get precise cuts
noprogress: Do not print the progress bar noprogress: Do not print the progress bar
live_from_start: Whether to download livestreams videos from the start live_from_start: Whether to download livestreams videos from the start
warn_when_outdated: Emit a warning if the yt-dlp version is older than 90 days
The following parameters are not used by YoutubeDL itself, they are used by The following parameters are not used by YoutubeDL itself, they are used by
the downloader (see yt_dlp/downloader/common.py): the downloader (see yt_dlp/downloader/common.py):
nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize, nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
max_filesize, test, noresizebuffer, retries, file_access_retries, fragment_retries, max_filesize, test, noresizebuffer, retries, file_access_retries, fragment_retries,
continuedl, xattr_set_filesize, hls_use_mpegts, http_chunk_size, continuedl, hls_use_mpegts, http_chunk_size, external_downloader_args,
external_downloader_args, concurrent_fragment_downloads, progress_delta. concurrent_fragment_downloads, progress_delta.
The following options are used by the post processors: The following options are used by the post processors:
ffmpeg_location: Location of the ffmpeg/avconv binary; either the path ffmpeg_location: Location of the ffmpeg binary; either the path
to the binary or its containing directory. to the binary or its containing directory.
postprocessor_args: A dictionary of postprocessor/executable keys (in lower case) postprocessor_args: A dictionary of postprocessor/executable keys (in lower case)
and a list of additional command-line arguments for the and a list of additional command-line arguments for the
@@ -528,6 +531,7 @@ class YoutubeDL:
discontinuities such as ad breaks (default: False) discontinuities such as ad breaks (default: False)
extractor_args: A dictionary of arguments to be passed to the extractors. extractor_args: A dictionary of arguments to be passed to the extractors.
See "EXTRACTOR ARGUMENTS" for details. See "EXTRACTOR ARGUMENTS" for details.
Argument values must always be a list of string(s).
E.g. {'youtube': {'skip': ['dash', 'hls']}} E.g. {'youtube': {'skip': ['dash', 'hls']}}
mark_watched: Mark videos watched (even with --simulate). Only for YouTube mark_watched: Mark videos watched (even with --simulate). Only for YouTube
@@ -561,32 +565,14 @@ class YoutubeDL:
allsubtitles: - Use subtitleslangs = ['all'] allsubtitles: - Use subtitleslangs = ['all']
Downloads all the subtitles of the video Downloads all the subtitles of the video
(requires writesubtitles or writeautomaticsub) (requires writesubtitles or writeautomaticsub)
include_ads: - Doesn't work
Download ads as well
call_home: - Not implemented
Boolean, true if we are allowed to contact the
yt-dlp servers for debugging.
post_hooks: - Register a custom postprocessor post_hooks: - Register a custom postprocessor
A list of functions that get called as the final step A list of functions that get called as the final step
for each video file, after all postprocessors have been for each video file, after all postprocessors have been
called. The filename will be passed as the only argument. called. The filename will be passed as the only argument.
hls_prefer_native: - Use external_downloader = {'m3u8': 'native'} or {'m3u8': 'ffmpeg'}. hls_prefer_native: - Use external_downloader = {'m3u8': 'native'} or {'m3u8': 'ffmpeg'}.
Use the native HLS downloader instead of ffmpeg/avconv Use the native HLS downloader instead of ffmpeg
if True, otherwise use ffmpeg/avconv if False, otherwise if True, otherwise use ffmpeg if False, otherwise
use downloader suggested by extractor if None. use downloader suggested by extractor if None.
prefer_ffmpeg: - avconv support is deprecated
If False, use avconv instead of ffmpeg if both are available,
otherwise prefer ffmpeg.
youtube_include_dash_manifest: - Use extractor_args
If True (default), DASH manifests and related
data will be downloaded and processed by extractor.
You can reduce network I/O by disabling it if you don't
care about DASH. (only for youtube)
youtube_include_hls_manifest: - Use extractor_args
If True (default), HLS manifests and related
data will be downloaded and processed by extractor.
You can reduce network I/O by disabling it if you don't
care about HLS. (only for youtube)
no_color: Same as `color='no_color'` no_color: Same as `color='no_color'`
no_overwrites: Same as `overwrites=False` no_overwrites: Same as `overwrites=False`
""" """
@@ -594,7 +580,7 @@ class YoutubeDL:
_NUMERIC_FIELDS = { _NUMERIC_FIELDS = {
'width', 'height', 'asr', 'audio_channels', 'fps', 'width', 'height', 'asr', 'audio_channels', 'fps',
'tbr', 'abr', 'vbr', 'filesize', 'filesize_approx', 'tbr', 'abr', 'vbr', 'filesize', 'filesize_approx',
'timestamp', 'release_timestamp', 'timestamp', 'release_timestamp', 'available_at',
'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count', 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
'average_rating', 'comment_count', 'age_limit', 'average_rating', 'comment_count', 'age_limit',
'start_time', 'end_time', 'start_time', 'end_time',
@@ -604,13 +590,13 @@ class YoutubeDL:
_format_fields = { _format_fields = {
# NB: Keep in sync with the docstring of extractor/common.py # NB: Keep in sync with the docstring of extractor/common.py
'url', 'manifest_url', 'manifest_stream_number', 'ext', 'format', 'format_id', 'format_note', 'url', 'manifest_url', 'manifest_stream_number', 'ext', 'format', 'format_id', 'format_note', 'available_at',
'width', 'height', 'aspect_ratio', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'audio_channels', 'width', 'height', 'aspect_ratio', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'audio_channels',
'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns', 'hls_media_playlist_data', 'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns', 'hls_media_playlist_data',
'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start', 'is_dash_periods', 'request_data', 'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start', 'is_dash_periods', 'request_data',
'preference', 'language', 'language_preference', 'quality', 'source_preference', 'cookies', 'preference', 'language', 'language_preference', 'quality', 'source_preference', 'cookies',
'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'extra_param_to_segment_url', 'extra_param_to_key_url', 'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'extra_param_to_segment_url', 'extra_param_to_key_url',
'hls_aes', 'downloader_options', 'page_url', 'app', 'play_path', 'tc_url', 'flash_version', 'hls_aes', 'downloader_options', 'impersonate', 'page_url', 'app', 'play_path', 'tc_url', 'flash_version',
'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time', 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time',
} }
_deprecated_multivalue_fields = { _deprecated_multivalue_fields = {
@@ -640,6 +626,7 @@ class YoutubeDL:
self._printed_messages = set() self._printed_messages = set()
self._first_webpage_request = True self._first_webpage_request = True
self._post_hooks = [] self._post_hooks = []
self._close_hooks = []
self._progress_hooks = [] self._progress_hooks = []
self._postprocessor_hooks = [] self._postprocessor_hooks = []
self._download_retcode = 0 self._download_retcode = 0
@@ -699,6 +686,9 @@ class YoutubeDL:
system_deprecation = _get_system_deprecation() system_deprecation = _get_system_deprecation()
if system_deprecation: if system_deprecation:
self.deprecated_feature(system_deprecation.replace('\n', '\n ')) self.deprecated_feature(system_deprecation.replace('\n', '\n '))
elif self.params.get('warn_when_outdated'):
if outdated_warning := _get_outdated_warning():
self.report_warning(outdated_warning)
if self.params.get('allow_unplayable_formats'): if self.params.get('allow_unplayable_formats'):
self.report_warning( self.report_warning(
@@ -741,12 +731,6 @@ class YoutubeDL:
return True return True
return False return False
if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
if self.params.get('geo_verification_proxy') is None:
self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
check_deprecated('useid', '--id', '-o "%(id)s.%(ext)s"') check_deprecated('useid', '--id', '-o "%(id)s.%(ext)s"')
for msg in self.params.get('_warnings', []): for msg in self.params.get('_warnings', []):
@@ -908,6 +892,11 @@ class YoutubeDL:
"""Add the post hook""" """Add the post hook"""
self._post_hooks.append(ph) self._post_hooks.append(ph)
def add_close_hook(self, ch):
"""Add a close hook, called when YoutubeDL.close() is called"""
assert callable(ch), 'Close hook must be callable'
self._close_hooks.append(ch)
def add_progress_hook(self, ph): def add_progress_hook(self, ph):
"""Add the download progress hook""" """Add the download progress hook"""
self._progress_hooks.append(ph) self._progress_hooks.append(ph)
@@ -1016,6 +1005,9 @@ class YoutubeDL:
self._request_director.close() self._request_director.close()
del self._request_director del self._request_director
for close_hook in self._close_hooks:
close_hook()
def trouble(self, message=None, tb=None, is_error=True): def trouble(self, message=None, tb=None, is_error=True):
"""Determine action to take when a download problem appears. """Determine action to take when a download problem appears.
@@ -2015,7 +2007,7 @@ class YoutubeDL:
else: else:
entries = resolved_entries = list(entries) entries = resolved_entries = list(entries)
n_entries = len(resolved_entries) n_entries = len(resolved_entries)
ie_result['requested_entries'], ie_result['entries'] = tuple(zip(*resolved_entries)) or ([], []) ie_result['requested_entries'], ie_result['entries'] = tuple(zip(*resolved_entries, strict=True)) or ([], [])
if not ie_result.get('playlist_count'): if not ie_result.get('playlist_count'):
# Better to do this after potentially exhausting entries # Better to do this after potentially exhausting entries
ie_result['playlist_count'] = all_entries.get_full_count() ie_result['playlist_count'] = all_entries.get_full_count()
@@ -2185,7 +2177,7 @@ class YoutubeDL:
return op(actual_value, comparison_value) return op(actual_value, comparison_value)
return _filter return _filter
def _check_formats(self, formats): def _check_formats(self, formats, warning=True):
for f in formats: for f in formats:
working = f.get('__working') working = f.get('__working')
if working is not None: if working is not None:
@@ -2198,6 +2190,9 @@ class YoutubeDL:
continue continue
temp_file = tempfile.NamedTemporaryFile(suffix='.tmp', delete=False, dir=path or None) temp_file = tempfile.NamedTemporaryFile(suffix='.tmp', delete=False, dir=path or None)
temp_file.close() temp_file.close()
# If FragmentFD fails when testing a fragment, it will wrongly set a non-zero return code.
# Save the actual return code for later. See https://github.com/yt-dlp/yt-dlp/issues/13750
original_retcode = self._download_retcode
try: try:
success, _ = self.dl(temp_file.name, f, test=True) success, _ = self.dl(temp_file.name, f, test=True)
except (DownloadError, OSError, ValueError, *network_exceptions): except (DownloadError, OSError, ValueError, *network_exceptions):
@@ -2208,11 +2203,18 @@ class YoutubeDL:
os.remove(temp_file.name) os.remove(temp_file.name)
except OSError: except OSError:
self.report_warning(f'Unable to delete temporary file "{temp_file.name}"') self.report_warning(f'Unable to delete temporary file "{temp_file.name}"')
# Restore the actual return code
self._download_retcode = original_retcode
f['__working'] = success f['__working'] = success
if success: if success:
f.pop('__needs_testing', None)
yield f yield f
else: else:
self.to_screen('[info] Unable to download format {}. Skipping...'.format(f['format_id'])) msg = f'Unable to download format {f["format_id"]}. Skipping...'
if warning:
self.report_warning(msg)
else:
self.to_screen(f'[info] {msg}')
def _select_formats(self, formats, selector): def _select_formats(self, formats, selector):
return list(selector({ return list(selector({
@@ -2692,11 +2694,7 @@ class YoutubeDL:
('modified_timestamp', 'modified_date'), ('modified_timestamp', 'modified_date'),
): ):
if info_dict.get(date_key) is None and info_dict.get(ts_key) is not None: if info_dict.get(date_key) is None and info_dict.get(ts_key) is not None:
# Working around out-of-range timestamp values (e.g. negative ones on Windows, info_dict[date_key] = strftime_or_none(info_dict[ts_key])
# see http://bugs.python.org/issue1646728)
with contextlib.suppress(ValueError, OverflowError, OSError):
upload_date = dt.datetime.fromtimestamp(info_dict[ts_key], dt.timezone.utc)
info_dict[date_key] = upload_date.strftime('%Y%m%d')
if not info_dict.get('release_year'): if not info_dict.get('release_year'):
info_dict['release_year'] = traverse_obj(info_dict, ('release_date', {lambda x: int(x[:4])})) info_dict['release_year'] = traverse_obj(info_dict, ('release_date', {lambda x: int(x[:4])}))
@@ -2787,7 +2785,7 @@ class YoutubeDL:
dummy_chapter = {'end_time': 0, 'start_time': info_dict.get('duration')} dummy_chapter = {'end_time': 0, 'start_time': info_dict.get('duration')}
for idx, (prev, current, next_) in enumerate(zip( for idx, (prev, current, next_) in enumerate(zip(
(dummy_chapter, *chapters), chapters, (*chapters[1:], dummy_chapter)), 1): (dummy_chapter, *chapters), chapters, (*chapters[1:], dummy_chapter), strict=False), 1):
if current.get('start_time') is None: if current.get('start_time') is None:
current['start_time'] = prev.get('end_time') current['start_time'] = prev.get('end_time')
if not current.get('end_time'): if not current.get('end_time'):
@@ -2938,7 +2936,7 @@ class YoutubeDL:
) )
if self.params.get('check_formats') is True: if self.params.get('check_formats') is True:
formats = LazyList(self._check_formats(formats[::-1]), reverse=True) formats = LazyList(self._check_formats(formats[::-1], warning=False), reverse=True)
if not formats or formats[0] is not info_dict: if not formats or formats[0] is not info_dict:
# only set the 'formats' fields if the original info_dict list them # only set the 'formats' fields if the original info_dict list them
@@ -3211,6 +3209,7 @@ class YoutubeDL:
} }
else: else:
params = self.params params = self.params
fd = get_suitable_downloader(info, params, to_stdout=(name == '-'))(self, params) fd = get_suitable_downloader(info, params, to_stdout=(name == '-'))(self, params)
if not test: if not test:
for ph in self._progress_hooks: for ph in self._progress_hooks:
@@ -3313,28 +3312,6 @@ class YoutubeDL:
elif _infojson_written is None: elif _infojson_written is None:
return return
# Note: Annotations are deprecated
annofn = None
if self.params.get('writeannotations', False):
annofn = self.prepare_filename(info_dict, 'annotation')
if annofn:
if not self._ensure_dir_exists(annofn):
return
if not self.params.get('overwrites', True) and os.path.exists(annofn):
self.to_screen('[info] Video annotations are already present')
elif not info_dict.get('annotations'):
self.report_warning('There are no annotations to write.')
else:
try:
self.to_screen('[info] Writing video annotations to: ' + annofn)
with open(annofn, 'w', encoding='utf-8') as annofile:
annofile.write(info_dict['annotations'])
except (KeyError, TypeError):
self.report_warning('There are no annotations to write.')
except OSError:
self.report_error('Cannot write annotations file: ' + annofn)
return
# Write internet shortcut files # Write internet shortcut files
def _write_link_file(link_type): def _write_link_file(link_type):
url = try_get(info_dict['webpage_url'], iri_to_uri) url = try_get(info_dict['webpage_url'], iri_to_uri)
@@ -3393,7 +3370,7 @@ class YoutubeDL:
def existing_video_file(*filepaths): def existing_video_file(*filepaths):
ext = info_dict.get('ext') ext = info_dict.get('ext')
converted = lambda file: replace_extension(file, self.params.get('final_ext') or ext, ext) converted = lambda file: replace_extension(file, self.params.get('final_ext') or ext, ext)
file = self.existing_file(itertools.chain(*zip(map(converted, filepaths), filepaths)), file = self.existing_file(itertools.chain(*zip(map(converted, filepaths), filepaths, strict=True)),
default_overwrite=False) default_overwrite=False)
if file: if file:
info_dict['ext'] = os.path.splitext(file)[1][1:] info_dict['ext'] = os.path.splitext(file)[1][1:]
@@ -3686,6 +3663,8 @@ class YoutubeDL:
return {k: filter_fn(v) for k, v in obj.items() if not reject(k, v)} return {k: filter_fn(v) for k, v in obj.items() if not reject(k, v)}
elif isinstance(obj, (list, tuple, set, LazyList)): elif isinstance(obj, (list, tuple, set, LazyList)):
return list(map(filter_fn, obj)) return list(map(filter_fn, obj))
elif isinstance(obj, ImpersonateTarget):
return str(obj)
elif obj is None or isinstance(obj, (str, int, float, bool)): elif obj is None or isinstance(obj, (str, int, float, bool)):
return obj return obj
else: else:
@@ -3954,6 +3933,7 @@ class YoutubeDL:
self._format_out('UNSUPPORTED', self.Styles.BAD_FORMAT) if f.get('ext') in ('f4f', 'f4m') else None, self._format_out('UNSUPPORTED', self.Styles.BAD_FORMAT) if f.get('ext') in ('f4f', 'f4m') else None,
(self._format_out('Maybe DRM', self.Styles.WARNING) if f.get('has_drm') == 'maybe' (self._format_out('Maybe DRM', self.Styles.WARNING) if f.get('has_drm') == 'maybe'
else self._format_out('DRM', self.Styles.BAD_FORMAT) if f.get('has_drm') else None), else self._format_out('DRM', self.Styles.BAD_FORMAT) if f.get('has_drm') else None),
self._format_out('Untested', self.Styles.WARNING) if f.get('__needs_testing') else None,
format_field(f, 'format_note'), format_field(f, 'format_note'),
format_field(f, 'container', ignore=(None, f.get('ext'))), format_field(f, 'container', ignore=(None, f.get('ext'))),
delim=', '), delim=' '), delim=', '), delim=' '),
@@ -3976,7 +3956,7 @@ class YoutubeDL:
def render_subtitles_table(self, video_id, subtitles): def render_subtitles_table(self, video_id, subtitles):
def _row(lang, formats): def _row(lang, formats):
exts, names = zip(*((f['ext'], f.get('name') or 'unknown') for f in reversed(formats))) exts, names = zip(*((f['ext'], f.get('name') or 'unknown') for f in reversed(formats)), strict=True)
if len(set(names)) == 1: if len(set(names)) == 1:
names = [] if names[0] == 'unknown' else names[:1] names = [] if names[0] == 'unknown' else names[:1]
return [lang, ', '.join(names), ', '.join(exts)] return [lang, ', '.join(names), ', '.join(exts)]
@@ -4015,8 +3995,7 @@ class YoutubeDL:
if os.environ.get('TERM', '').lower() == 'dumb': if os.environ.get('TERM', '').lower() == 'dumb':
additional_info.append('dumb') additional_info.append('dumb')
if not supports_terminal_sequences(stream): if not supports_terminal_sequences(stream):
from .utils import WINDOWS_VT_MODE # Must be imported locally additional_info.append('No VT' if WINDOWS_VT_MODE.value is False else 'No ANSI')
additional_info.append('No VT' if WINDOWS_VT_MODE is False else 'No ANSI')
if additional_info: if additional_info:
ret = f'{ret} ({",".join(additional_info)})' ret = f'{ret} ({",".join(additional_info)})'
return ret return ret
@@ -4133,8 +4112,7 @@ class YoutubeDL:
self.params.get('cookiefile'), self.params.get('cookiesfrombrowser'), self) self.params.get('cookiefile'), self.params.get('cookiesfrombrowser'), self)
except CookieLoadError as error: except CookieLoadError as error:
cause = error.__context__ cause = error.__context__
# compat: <=py3.9: `traceback.format_exception` has a different signature self.report_error(str(cause), tb=''.join(traceback.format_exception(cause)))
self.report_error(str(cause), tb=''.join(traceback.format_exception(None, cause, cause.__traceback__)))
raise raise
@property @property
@@ -4162,6 +4140,31 @@ class YoutubeDL:
for rh in self._request_director.handlers.values() for rh in self._request_director.handlers.values()
if isinstance(rh, ImpersonateRequestHandler)) if isinstance(rh, ImpersonateRequestHandler))
def _parse_impersonate_targets(self, impersonate):
if impersonate in (True, ''):
impersonate = ImpersonateTarget()
requested_targets = [
t if isinstance(t, ImpersonateTarget) else ImpersonateTarget.from_str(t)
for t in variadic(impersonate)
] if impersonate else []
available_target = next(filter(self._impersonate_target_available, requested_targets), None)
return available_target, requested_targets
@staticmethod
def _unavailable_targets_message(requested_targets, note=None, is_error=False):
note = note or 'The extractor specified to use impersonation for this download'
specific_targets = ', '.join(filter(None, map(str, requested_targets)))
message = (
'no impersonate target is available' if not specific_targets
else f'none of these impersonate targets are available: {specific_targets}')
return (
f'{note}, but {message}. {"See" if is_error else "If you encounter errors, then see"}'
f' https://github.com/yt-dlp/yt-dlp#impersonation '
f'for information on installing the required dependencies')
def urlopen(self, req): def urlopen(self, req):
""" Start an HTTP download """ """ Start an HTTP download """
if isinstance(req, str): if isinstance(req, str):
plugins/youtube_download/yt_dlp/__init__.py
+27 -61
View File
@@ -1,8 +1,8 @@
import sys import sys
if sys.version_info < (3, 9): if sys.version_info < (3, 10):
raise ImportError( raise ImportError(
f'You are using an unsupported version of Python. Only Python versions 3.9 and above are supported by yt-dlp') # noqa: F541 f'You are using an unsupported version of Python. Only Python versions 3.10 and above are supported by yt-dlp') # noqa: F541
__license__ = 'The Unlicense' __license__ = 'The Unlicense'
@@ -59,11 +59,9 @@ from .utils import (
render_table, render_table,
setproctitle, setproctitle,
shell_quote, shell_quote,
traverse_obj,
variadic, variadic,
write_string, write_string,
) )
from .utils.networking import std_headers
from .utils._utils import _UnsafeExtensionError from .utils._utils import _UnsafeExtensionError
from .YoutubeDL import YoutubeDL from .YoutubeDL import YoutubeDL
@@ -157,7 +155,13 @@ def set_compat_opts(opts):
if 'format-sort' in opts.compat_opts: if 'format-sort' in opts.compat_opts:
opts.format_sort.extend(FormatSorter.ytdl_default) opts.format_sort.extend(FormatSorter.ytdl_default)
elif 'prefer-vp9-sort' in opts.compat_opts: elif 'prefer-vp9-sort' in opts.compat_opts:
opts.format_sort.extend(FormatSorter._prefer_vp9_sort) FormatSorter.default = FormatSorter._prefer_vp9_sort
if 'mtime-by-default' in opts.compat_opts:
if opts.updatetime is None:
opts.updatetime = True
else:
_unused_compat_opt('mtime-by-default')
_video_multistreams_set = set_default_compat('multistreams', 'allow_multiple_video_streams', False, remove_compat=False) _video_multistreams_set = set_default_compat('multistreams', 'allow_multiple_video_streams', False, remove_compat=False)
_audio_multistreams_set = set_default_compat('multistreams', 'allow_multiple_audio_streams', False, remove_compat=False) _audio_multistreams_set = set_default_compat('multistreams', 'allow_multiple_audio_streams', False, remove_compat=False)
@@ -494,6 +498,14 @@ def validate_options(opts):
'To let yt-dlp download and merge the best available formats, simply do not pass any format selection', 'To let yt-dlp download and merge the best available formats, simply do not pass any format selection',
'If you know what you are doing and want only the best pre-merged format, use "-f b" instead to suppress this warning'))) 'If you know what you are doing and want only the best pre-merged format, use "-f b" instead to suppress this warning')))
# Common mistake: -f mp4
if opts.format == 'mp4':
warnings.append('.\n '.join((
'"-f mp4" selects the best pre-merged mp4 format which is often not what\'s intended',
'Pre-merged mp4 formats are not available from all sites, or may only be available in lower quality',
'To prioritize the best h264 video and aac audio in an mp4 container, use "-t mp4" instead',
'If you know what you are doing and want a pre-merged mp4 format, use "-f b[ext=mp4]" instead to suppress this warning')))
# --(postprocessor/downloader)-args without name # --(postprocessor/downloader)-args without name
def report_args_compat(name, value, key1, key2=None, where=None): def report_args_compat(name, value, key1, key2=None, where=None):
if key1 in value and key2 not in value: if key1 in value and key2 not in value:
@@ -509,7 +521,6 @@ def validate_options(opts):
if report_args_compat('post-processor', opts.postprocessor_args, 'default-compat', 'default'): if report_args_compat('post-processor', opts.postprocessor_args, 'default-compat', 'default'):
opts.postprocessor_args['default'] = opts.postprocessor_args.pop('default-compat') opts.postprocessor_args['default'] = opts.postprocessor_args.pop('default-compat')
opts.postprocessor_args.setdefault('sponskrub', [])
def report_conflict(arg1, opt1, arg2='--allow-unplayable-formats', opt2='allow_unplayable_formats', def report_conflict(arg1, opt1, arg2='--allow-unplayable-formats', opt2='allow_unplayable_formats',
val1=NO_DEFAULT, val2=NO_DEFAULT, default=False): val1=NO_DEFAULT, val2=NO_DEFAULT, default=False):
@@ -534,11 +545,6 @@ def validate_options(opts):
'"--exec before_dl:"', 'exec_cmd', val2=opts.exec_cmd.get('before_dl')) '"--exec before_dl:"', 'exec_cmd', val2=opts.exec_cmd.get('before_dl'))
report_conflict('--id', 'useid', '--output', 'outtmpl', val2=opts.outtmpl.get('default')) report_conflict('--id', 'useid', '--output', 'outtmpl', val2=opts.outtmpl.get('default'))
report_conflict('--remux-video', 'remuxvideo', '--recode-video', 'recodevideo') report_conflict('--remux-video', 'remuxvideo', '--recode-video', 'recodevideo')
report_conflict('--sponskrub', 'sponskrub', '--remove-chapters', 'remove_chapters')
report_conflict('--sponskrub', 'sponskrub', '--sponsorblock-mark', 'sponsorblock_mark')
report_conflict('--sponskrub', 'sponskrub', '--sponsorblock-remove', 'sponsorblock_remove')
report_conflict('--sponskrub-cut', 'sponskrub_cut', '--split-chapter', 'split_chapters',
val1=opts.sponskrub and opts.sponskrub_cut)
# Conflicts with --allow-unplayable-formats # Conflicts with --allow-unplayable-formats
report_conflict('--embed-metadata', 'addmetadata') report_conflict('--embed-metadata', 'addmetadata')
@@ -551,23 +557,15 @@ def validate_options(opts):
report_conflict('--recode-video', 'recodevideo') report_conflict('--recode-video', 'recodevideo')
report_conflict('--remove-chapters', 'remove_chapters', default=[]) report_conflict('--remove-chapters', 'remove_chapters', default=[])
report_conflict('--remux-video', 'remuxvideo') report_conflict('--remux-video', 'remuxvideo')
report_conflict('--sponskrub', 'sponskrub')
report_conflict('--sponsorblock-remove', 'sponsorblock_remove', default=set()) report_conflict('--sponsorblock-remove', 'sponsorblock_remove', default=set())
report_conflict('--xattrs', 'xattrs') report_conflict('--xattrs', 'xattrs')
# Fully deprecated options if hasattr(opts, '_deprecated_options'):
def report_deprecation(val, old, new=None):
if not val:
return
deprecation_warnings.append( deprecation_warnings.append(
f'{old} is deprecated and may be removed in a future version. Use {new} instead' if new f'The following options have been deprecated: {", ".join(opts._deprecated_options)}\n'
else f'{old} is deprecated and may not work as expected') 'Please remove them from your command/configuration to avoid future errors.\n'
'See https://github.com/yt-dlp/yt-dlp/issues/14198 for more details')
report_deprecation(opts.sponskrub, '--sponskrub', '--sponsorblock-mark or --sponsorblock-remove') del opts._deprecated_options
report_deprecation(not opts.prefer_ffmpeg, '--prefer-avconv', 'ffmpeg')
# report_deprecation(opts.include_ads, '--include-ads') # We may re-implement this in future
# report_deprecation(opts.call_home, '--call-home') # We may re-implement this in future
# report_deprecation(opts.writeannotations, '--write-annotations') # It's just that no website has it
# Dependent options # Dependent options
opts.date = DateRange.day(opts.date) if opts.date else DateRange(opts.dateafter, opts.datebefore) opts.date = DateRange.day(opts.date) if opts.date else DateRange(opts.dateafter, opts.datebefore)
@@ -698,21 +696,6 @@ def get_postprocessors(opts):
'add_metadata': opts.addmetadata, 'add_metadata': opts.addmetadata,
'add_infojson': opts.embed_infojson, 'add_infojson': opts.embed_infojson,
} }
# Deprecated
# This should be above EmbedThumbnail since sponskrub removes the thumbnail attachment
# but must be below EmbedSubtitle and FFmpegMetadata
# See https://github.com/yt-dlp/yt-dlp/issues/204 , https://github.com/faissaloo/SponSkrub/issues/29
# If opts.sponskrub is None, sponskrub is used, but it silently fails if the executable can't be found
if opts.sponskrub is not False:
yield {
'key': 'SponSkrub',
'path': opts.sponskrub_path,
'args': opts.sponskrub_args,
'cut': opts.sponskrub_cut,
'force': opts.sponskrub_force,
'ignoreerror': opts.sponskrub is None,
'_from_cli': True,
}
if opts.embedthumbnail: if opts.embedthumbnail:
yield { yield {
'key': 'EmbedThumbnail', 'key': 'EmbedThumbnail',
@@ -871,7 +854,6 @@ def parse_options(argv=None):
'nopart': opts.nopart, 'nopart': opts.nopart,
'updatetime': opts.updatetime, 'updatetime': opts.updatetime,
'writedescription': opts.writedescription, 'writedescription': opts.writedescription,
'writeannotations': opts.writeannotations,
'writeinfojson': opts.writeinfojson, 'writeinfojson': opts.writeinfojson,
'allow_playlist_files': opts.allow_playlist_files, 'allow_playlist_files': opts.allow_playlist_files,
'clean_infojson': opts.clean_infojson, 'clean_infojson': opts.clean_infojson,
@@ -905,7 +887,6 @@ def parse_options(argv=None):
'max_views': opts.max_views, 'max_views': opts.max_views,
'daterange': opts.date, 'daterange': opts.date,
'cachedir': opts.cachedir, 'cachedir': opts.cachedir,
'youtube_print_sig_code': opts.youtube_print_sig_code,
'age_limit': opts.age_limit, 'age_limit': opts.age_limit,
'download_archive': opts.download_archive, 'download_archive': opts.download_archive,
'break_on_existing': opts.break_on_existing, 'break_on_existing': opts.break_on_existing,
@@ -923,13 +904,9 @@ def parse_options(argv=None):
'socket_timeout': opts.socket_timeout, 'socket_timeout': opts.socket_timeout,
'bidi_workaround': opts.bidi_workaround, 'bidi_workaround': opts.bidi_workaround,
'debug_printtraffic': opts.debug_printtraffic, 'debug_printtraffic': opts.debug_printtraffic,
'prefer_ffmpeg': opts.prefer_ffmpeg,
'include_ads': opts.include_ads,
'default_search': opts.default_search, 'default_search': opts.default_search,
'dynamic_mpd': opts.dynamic_mpd, 'dynamic_mpd': opts.dynamic_mpd,
'extractor_args': opts.extractor_args, 'extractor_args': opts.extractor_args,
'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
'youtube_include_hls_manifest': opts.youtube_include_hls_manifest,
'encoding': opts.encoding, 'encoding': opts.encoding,
'extract_flat': opts.extract_flat, 'extract_flat': opts.extract_flat,
'live_from_start': opts.live_from_start, 'live_from_start': opts.live_from_start,
@@ -941,7 +918,6 @@ def parse_options(argv=None):
'fixup': opts.fixup, 'fixup': opts.fixup,
'source_address': opts.source_address, 'source_address': opts.source_address,
'impersonate': opts.impersonate, 'impersonate': opts.impersonate,
'call_home': opts.call_home,
'sleep_interval_requests': opts.sleep_interval_requests, 'sleep_interval_requests': opts.sleep_interval_requests,
'sleep_interval': opts.sleep_interval, 'sleep_interval': opts.sleep_interval,
'max_sleep_interval': opts.max_sleep_interval, 'max_sleep_interval': opts.max_sleep_interval,
@@ -951,7 +927,6 @@ def parse_options(argv=None):
'force_keyframes_at_cuts': opts.force_keyframes_at_cuts, 'force_keyframes_at_cuts': opts.force_keyframes_at_cuts,
'list_thumbnails': opts.list_thumbnails, 'list_thumbnails': opts.list_thumbnails,
'playlist_items': opts.playlist_items, 'playlist_items': opts.playlist_items,
'xattr_set_filesize': opts.xattr_set_filesize,
'match_filter': opts.match_filter, 'match_filter': opts.match_filter,
'color': opts.color, 'color': opts.color,
'ffmpeg_location': opts.ffmpeg_location, 'ffmpeg_location': opts.ffmpeg_location,
@@ -960,11 +935,12 @@ def parse_options(argv=None):
'hls_split_discontinuity': opts.hls_split_discontinuity, 'hls_split_discontinuity': opts.hls_split_discontinuity,
'external_downloader_args': opts.external_downloader_args, 'external_downloader_args': opts.external_downloader_args,
'postprocessor_args': opts.postprocessor_args, 'postprocessor_args': opts.postprocessor_args,
'cn_verification_proxy': opts.cn_verification_proxy,
'geo_verification_proxy': opts.geo_verification_proxy, 'geo_verification_proxy': opts.geo_verification_proxy,
'geo_bypass': opts.geo_bypass, 'geo_bypass': opts.geo_bypass,
'geo_bypass_country': opts.geo_bypass_country, 'geo_bypass_country': opts.geo_bypass_country,
'geo_bypass_ip_block': opts.geo_bypass_ip_block, 'geo_bypass_ip_block': opts.geo_bypass_ip_block,
'useid': opts.useid or None,
'warn_when_outdated': opts.update_self is None,
'_warnings': warnings, '_warnings': warnings,
'_deprecation_warnings': deprecation_warnings, '_deprecation_warnings': deprecation_warnings,
'compat_opts': opts.compat_opts, 'compat_opts': opts.compat_opts,
@@ -976,12 +952,6 @@ def _real_main(argv=None):
parser, opts, all_urls, ydl_opts = parse_options(argv) parser, opts, all_urls, ydl_opts = parse_options(argv)
# Dump user agent
if opts.dump_user_agent:
ua = traverse_obj(opts.headers, 'User-Agent', casesense=False, default=std_headers['User-Agent'])
write_string(f'{ua}\n', out=sys.stdout)
return
if print_extractor_information(opts, all_urls): if print_extractor_information(opts, all_urls):
return return
@@ -1004,13 +974,8 @@ def _real_main(argv=None):
try: try:
updater = Updater(ydl, opts.update_self) updater = Updater(ydl, opts.update_self)
if opts.update_self and updater.update() and actual_use: if opts.update_self and updater.update() and actual_use and updater.cmd:
if updater.cmd: return updater.restart()
return updater.restart()
# This code is reachable only for zip variant in py < 3.10
# It makes sense to exit here, but the old behavior is to continue
ydl.report_warning('Restart yt-dlp to use the updated version')
# return 100, 'ERROR: The program must exit for the update to complete'
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
ydl._download_retcode = 100 ydl._download_retcode = 100
@@ -1024,6 +989,7 @@ def _real_main(argv=None):
(ImpersonateTarget('safari'), 'curl_cffi'), (ImpersonateTarget('safari'), 'curl_cffi'),
(ImpersonateTarget('firefox'), 'curl_cffi>=0.10'), (ImpersonateTarget('firefox'), 'curl_cffi>=0.10'),
(ImpersonateTarget('edge'), 'curl_cffi'), (ImpersonateTarget('edge'), 'curl_cffi'),
(ImpersonateTarget('tor'), 'curl_cffi>=0.11'),
] ]
available_targets = ydl._get_available_impersonate_targets() available_targets = ydl._get_available_impersonate_targets()
plugins/youtube_download/yt_dlp/aes.py
+2 -2
View File
@@ -435,7 +435,7 @@ def sub_bytes_inv(data):
def rotate(data): def rotate(data):
return data[1:] + [data[0]] return [*data[1:], data[0]]
def key_schedule_core(data, rcon_iteration): def key_schedule_core(data, rcon_iteration):
@@ -447,7 +447,7 @@ def key_schedule_core(data, rcon_iteration):
def xor(data1, data2): def xor(data1, data2):
return [x ^ y for x, y in zip(data1, data2)] return [x ^ y for x, y in zip(data1, data2, strict=False)]
def iter_mix_columns(data, matrix): def iter_mix_columns(data, matrix):
plugins/youtube_download/yt_dlp/compat/__init__.py
+8
View File
@@ -1,3 +1,4 @@
import datetime as dt
import os import os
import xml.etree.ElementTree as etree import xml.etree.ElementTree as etree
@@ -27,6 +28,13 @@ def compat_ord(c):
return c if isinstance(c, int) else ord(c) return c if isinstance(c, int) else ord(c)
def compat_datetime_from_timestamp(timestamp):
# Calling dt.datetime.fromtimestamp with negative timestamps throws error in Windows
# Ref: https://github.com/yt-dlp/yt-dlp/issues/5185, https://github.com/python/cpython/issues/81708,
# https://github.com/yt-dlp/yt-dlp/issues/6706#issuecomment-1496842642
return (dt.datetime.fromtimestamp(0, dt.timezone.utc) + dt.timedelta(seconds=timestamp))
# Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl # Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl
# See https://github.com/yt-dlp/yt-dlp/issues/792 # See https://github.com/yt-dlp/yt-dlp/issues/792
# https://docs.python.org/3/library/os.path.html#os.path.expanduser # https://docs.python.org/3/library/os.path.html#os.path.expanduser
plugins/youtube_download/yt_dlp/compat/_legacy.py
+1 -1
View File
@@ -37,7 +37,7 @@ from ..dependencies import websockets as compat_websockets # noqa: F401
from ..dependencies.Cryptodome import AES as compat_pycrypto_AES # noqa: F401 from ..dependencies.Cryptodome import AES as compat_pycrypto_AES # noqa: F401
from ..networking.exceptions import HTTPError as compat_HTTPError from ..networking.exceptions import HTTPError as compat_HTTPError
passthrough_module(__name__, '...utils', ('WINDOWS_VT_MODE', 'windows_enable_vt_mode')) passthrough_module(__name__, '...utils', ('windows_enable_vt_mode',))
# compat_ctypes_WINFUNCTYPE = ctypes.WINFUNCTYPE # compat_ctypes_WINFUNCTYPE = ctypes.WINFUNCTYPE
plugins/youtube_download/yt_dlp/compat/types.py
-13
View File
@@ -1,13 +0,0 @@
# flake8: noqa: F405
from types import * # noqa: F403
from .compat_utils import passthrough_module
passthrough_module(__name__, 'types')
del passthrough_module
try:
# NB: pypy has builtin NoneType, so checking NameError won't work
from types import NoneType # >= 3.10
except ImportError:
NoneType = type(None)
plugins/youtube_download/yt_dlp/compat/urllib/request.py
+4 -8
View File
@@ -22,15 +22,11 @@ if os.name == 'nt':
def getproxies_registry_patched(): def getproxies_registry_patched():
proxies = getproxies_registry() proxies = getproxies_registry()
if (
sys.version_info >= (3, 10, 5) # https://docs.python.org/3.10/whatsnew/changelog.html#python-3-10-5-final
or (3, 9, 13) <= sys.version_info < (3, 10) # https://docs.python.org/3.9/whatsnew/changelog.html#python-3-9-13-final
):
return proxies
for scheme in ('https', 'ftp'): if sys.version_info < (3, 10, 5): # https://docs.python.org/3.10/whatsnew/changelog.html#python-3-10-5-final
if scheme in proxies and proxies[scheme].startswith(f'{scheme}://'): for scheme in ('https', 'ftp'):
proxies[scheme] = 'http' + proxies[scheme][len(scheme):] if scheme in proxies and proxies[scheme].startswith(f'{scheme}://'):
proxies[scheme] = 'http' + proxies[scheme][len(scheme):]
return proxies return proxies
plugins/youtube_download/yt_dlp/cookies.py
+38 -34
View File
@@ -125,6 +125,8 @@ def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(),
def _extract_firefox_cookies(profile, container, logger): def _extract_firefox_cookies(profile, container, logger):
MAX_SUPPORTED_DB_SCHEMA_VERSION = 16
logger.info('Extracting cookies from firefox') logger.info('Extracting cookies from firefox')
if not sqlite3: if not sqlite3:
logger.warning('Cannot extract cookies from firefox without sqlite3 support. ' logger.warning('Cannot extract cookies from firefox without sqlite3 support. '
@@ -159,9 +161,11 @@ def _extract_firefox_cookies(profile, container, logger):
raise ValueError(f'could not find firefox container "{container}" in containers.json') raise ValueError(f'could not find firefox container "{container}" in containers.json')
with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir: with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
cursor = None cursor = _open_database_copy(cookie_database_path, tmpdir)
try: with contextlib.closing(cursor.connection):
cursor = _open_database_copy(cookie_database_path, tmpdir) db_schema_version = cursor.execute('PRAGMA user_version;').fetchone()[0]
if db_schema_version > MAX_SUPPORTED_DB_SCHEMA_VERSION:
logger.warning(f'Possibly unsupported firefox cookies database version: {db_schema_version}')
if isinstance(container_id, int): if isinstance(container_id, int):
logger.debug( logger.debug(
f'Only loading cookies from firefox container "{container}", ID {container_id}') f'Only loading cookies from firefox container "{container}", ID {container_id}')
@@ -180,6 +184,10 @@ def _extract_firefox_cookies(profile, container, logger):
total_cookie_count = len(table) total_cookie_count = len(table)
for i, (host, name, value, path, expiry, is_secure) in enumerate(table): for i, (host, name, value, path, expiry, is_secure) in enumerate(table):
progress_bar.print(f'Loading cookie {i: 6d}/{total_cookie_count: 6d}') progress_bar.print(f'Loading cookie {i: 6d}/{total_cookie_count: 6d}')
# FF142 upgraded cookies DB to schema version 16 and started using milliseconds for cookie expiry
# Ref: https://github.com/mozilla-firefox/firefox/commit/5869af852cd20425165837f6c2d9971f3efba83d
if db_schema_version >= 16 and expiry is not None:
expiry /= 1000
cookie = http.cookiejar.Cookie( cookie = http.cookiejar.Cookie(
version=0, name=name, value=value, port=None, port_specified=False, version=0, name=name, value=value, port=None, port_specified=False,
domain=host, domain_specified=bool(host), domain_initial_dot=host.startswith('.'), domain=host, domain_specified=bool(host), domain_initial_dot=host.startswith('.'),
@@ -188,9 +196,6 @@ def _extract_firefox_cookies(profile, container, logger):
jar.set_cookie(cookie) jar.set_cookie(cookie)
logger.info(f'Extracted {len(jar)} cookies from firefox') logger.info(f'Extracted {len(jar)} cookies from firefox')
return jar return jar
finally:
if cursor is not None:
cursor.connection.close()
def _firefox_browser_dirs(): def _firefox_browser_dirs():
@@ -764,11 +769,11 @@ def _get_linux_desktop_environment(env, logger):
GetDesktopEnvironment GetDesktopEnvironment
""" """
xdg_current_desktop = env.get('XDG_CURRENT_DESKTOP', None) xdg_current_desktop = env.get('XDG_CURRENT_DESKTOP', None)
desktop_session = env.get('DESKTOP_SESSION', None) desktop_session = env.get('DESKTOP_SESSION', '')
if xdg_current_desktop is not None: if xdg_current_desktop is not None:
for part in map(str.strip, xdg_current_desktop.split(':')): for part in map(str.strip, xdg_current_desktop.split(':')):
if part == 'Unity': if part == 'Unity':
if desktop_session is not None and 'gnome-fallback' in desktop_session: if 'gnome-fallback' in desktop_session:
return _LinuxDesktopEnvironment.GNOME return _LinuxDesktopEnvironment.GNOME
else: else:
return _LinuxDesktopEnvironment.UNITY return _LinuxDesktopEnvironment.UNITY
@@ -797,35 +802,34 @@ def _get_linux_desktop_environment(env, logger):
return _LinuxDesktopEnvironment.UKUI return _LinuxDesktopEnvironment.UKUI
elif part == 'LXQt': elif part == 'LXQt':
return _LinuxDesktopEnvironment.LXQT return _LinuxDesktopEnvironment.LXQT
logger.info(f'XDG_CURRENT_DESKTOP is set to an unknown value: "{xdg_current_desktop}"') logger.debug(f'XDG_CURRENT_DESKTOP is set to an unknown value: "{xdg_current_desktop}"')
elif desktop_session is not None: if desktop_session == 'deepin':
if desktop_session == 'deepin': return _LinuxDesktopEnvironment.DEEPIN
return _LinuxDesktopEnvironment.DEEPIN elif desktop_session in ('mate', 'gnome'):
elif desktop_session in ('mate', 'gnome'): return _LinuxDesktopEnvironment.GNOME
return _LinuxDesktopEnvironment.GNOME elif desktop_session in ('kde4', 'kde-plasma'):
elif desktop_session in ('kde4', 'kde-plasma'): return _LinuxDesktopEnvironment.KDE4
elif desktop_session == 'kde':
if 'KDE_SESSION_VERSION' in env:
return _LinuxDesktopEnvironment.KDE4 return _LinuxDesktopEnvironment.KDE4
elif desktop_session == 'kde':
if 'KDE_SESSION_VERSION' in env:
return _LinuxDesktopEnvironment.KDE4
else:
return _LinuxDesktopEnvironment.KDE3
elif 'xfce' in desktop_session or desktop_session == 'xubuntu':
return _LinuxDesktopEnvironment.XFCE
elif desktop_session == 'ukui':
return _LinuxDesktopEnvironment.UKUI
else: else:
logger.info(f'DESKTOP_SESSION is set to an unknown value: "{desktop_session}"') return _LinuxDesktopEnvironment.KDE3
elif 'xfce' in desktop_session or desktop_session == 'xubuntu':
return _LinuxDesktopEnvironment.XFCE
elif desktop_session == 'ukui':
return _LinuxDesktopEnvironment.UKUI
else: else:
if 'GNOME_DESKTOP_SESSION_ID' in env: logger.debug(f'DESKTOP_SESSION is set to an unknown value: "{desktop_session}"')
return _LinuxDesktopEnvironment.GNOME
elif 'KDE_FULL_SESSION' in env: if 'GNOME_DESKTOP_SESSION_ID' in env:
if 'KDE_SESSION_VERSION' in env: return _LinuxDesktopEnvironment.GNOME
return _LinuxDesktopEnvironment.KDE4 elif 'KDE_FULL_SESSION' in env:
else: if 'KDE_SESSION_VERSION' in env:
return _LinuxDesktopEnvironment.KDE3 return _LinuxDesktopEnvironment.KDE4
else:
return _LinuxDesktopEnvironment.KDE3
return _LinuxDesktopEnvironment.OTHER return _LinuxDesktopEnvironment.OTHER
@@ -1336,7 +1340,7 @@ class YoutubeDLCookieJar(http.cookiejar.MozillaCookieJar):
if len(cookie_list) != self._ENTRY_LEN: if len(cookie_list) != self._ENTRY_LEN:
raise http.cookiejar.LoadError(f'invalid length {len(cookie_list)}') raise http.cookiejar.LoadError(f'invalid length {len(cookie_list)}')
cookie = self._CookieFileEntry(*cookie_list) cookie = self._CookieFileEntry(*cookie_list)
if cookie.expires_at and not cookie.expires_at.isdigit(): if cookie.expires_at and not re.fullmatch(r'[0-9]+(?:\.[0-9]+)?', cookie.expires_at):
raise http.cookiejar.LoadError(f'invalid expires at {cookie.expires_at}') raise http.cookiejar.LoadError(f'invalid expires at {cookie.expires_at}')
return line return line
plugins/youtube_download/yt_dlp/downloader/__init__.py
+1 -1
View File
@@ -99,7 +99,7 @@ def _get_suitable_downloader(info_dict, protocol, params, default):
if external_downloader is None: if external_downloader is None:
if info_dict['to_stdout'] and FFmpegFD.can_merge_formats(info_dict, params): if info_dict['to_stdout'] and FFmpegFD.can_merge_formats(info_dict, params):
return FFmpegFD return FFmpegFD
elif external_downloader.lower() != 'native': elif external_downloader.lower() != 'native' and info_dict.get('impersonate') is None:
ed = get_external_downloader(external_downloader) ed = get_external_downloader(external_downloader)
if ed.can_download(info_dict, external_downloader): if ed.can_download(info_dict, external_downloader):
return ed return ed
plugins/youtube_download/yt_dlp/downloader/common.py
+25 -3
View File
@@ -62,7 +62,6 @@ class FileDownloader:
test: Download only first bytes to test the downloader. test: Download only first bytes to test the downloader.
min_filesize: Skip files smaller than this size min_filesize: Skip files smaller than this size
max_filesize: Skip files larger than this size max_filesize: Skip files larger than this size
xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
progress_delta: The minimum time between progress output, in seconds progress_delta: The minimum time between progress output, in seconds
external_downloader_args: A dictionary of downloader keys (in lower case) external_downloader_args: A dictionary of downloader keys (in lower case)
and a list of additional command-line arguments for the and a list of additional command-line arguments for the
@@ -455,14 +454,26 @@ class FileDownloader:
self._finish_multiline_status() self._finish_multiline_status()
return True, False return True, False
sleep_note = ''
if subtitle: if subtitle:
sleep_interval = self.params.get('sleep_interval_subtitles') or 0 sleep_interval = self.params.get('sleep_interval_subtitles') or 0
else: else:
min_sleep_interval = self.params.get('sleep_interval') or 0 min_sleep_interval = self.params.get('sleep_interval') or 0
max_sleep_interval = self.params.get('max_sleep_interval') or 0
if available_at := info_dict.get('available_at'):
forced_sleep_interval = available_at - int(time.time())
if forced_sleep_interval > min_sleep_interval:
sleep_note = 'as required by the site'
min_sleep_interval = forced_sleep_interval
if forced_sleep_interval > max_sleep_interval:
max_sleep_interval = forced_sleep_interval
sleep_interval = random.uniform( sleep_interval = random.uniform(
min_sleep_interval, self.params.get('max_sleep_interval') or min_sleep_interval) min_sleep_interval, max_sleep_interval or min_sleep_interval)
if sleep_interval > 0: if sleep_interval > 0:
self.to_screen(f'[download] Sleeping {sleep_interval:.2f} seconds ...') self.to_screen(f'[download] Sleeping {sleep_interval:.2f} seconds {sleep_note}...')
time.sleep(sleep_interval) time.sleep(sleep_interval)
ret = self.real_download(filename, info_dict) ret = self.real_download(filename, info_dict)
@@ -495,3 +506,14 @@ class FileDownloader:
exe = os.path.basename(args[0]) exe = os.path.basename(args[0])
self.write_debug(f'{exe} command line: {shell_quote(args)}') self.write_debug(f'{exe} command line: {shell_quote(args)}')
def _get_impersonate_target(self, info_dict):
impersonate = info_dict.get('impersonate')
if impersonate is None:
return None
available_target, requested_targets = self.ydl._parse_impersonate_targets(impersonate)
if available_target:
return available_target
elif requested_targets:
self.report_warning(self.ydl._unavailable_targets_message(requested_targets))
return None
plugins/youtube_download/yt_dlp/downloader/dash.py
+6 -1
View File
@@ -3,7 +3,7 @@ import urllib.parse
from . import get_suitable_downloader from . import get_suitable_downloader
from .fragment import FragmentFD from .fragment import FragmentFD
from ..utils import update_url_query, urljoin from ..utils import ReExtractInfo, update_url_query, urljoin
class DashSegmentsFD(FragmentFD): class DashSegmentsFD(FragmentFD):
@@ -28,6 +28,11 @@ class DashSegmentsFD(FragmentFD):
requested_formats = [{**info_dict, **fmt} for fmt in info_dict.get('requested_formats', [])] requested_formats = [{**info_dict, **fmt} for fmt in info_dict.get('requested_formats', [])]
args = [] args = []
for fmt in requested_formats or [info_dict]: for fmt in requested_formats or [info_dict]:
# Re-extract if --load-info-json is used and 'fragments' was originally a generator
# See https://github.com/yt-dlp/yt-dlp/issues/13906
if isinstance(fmt['fragments'], str):
raise ReExtractInfo('the stream needs to be re-extracted', expected=True)
try: try:
fragment_count = 1 if self.params.get('test') else len(fmt['fragments']) fragment_count = 1 if self.params.get('test') else len(fmt['fragments'])
except TypeError: except TypeError:
plugins/youtube_download/yt_dlp/downloader/external.py
+16 -6
View File
@@ -563,7 +563,7 @@ class FFmpegFD(ExternalFD):
f'{cookie.name}={cookie.value}; path={cookie.path}; domain={cookie.domain};\r\n' f'{cookie.name}={cookie.value}; path={cookie.path}; domain={cookie.domain};\r\n'
for cookie in cookies)]) for cookie in cookies)])
if fmt.get('http_headers') and is_http: if fmt.get('http_headers') and is_http:
# Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv: # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg:
# [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header. # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
args.extend(['-headers', ''.join(f'{key}: {val}\r\n' for key, val in fmt['http_headers'].items())]) args.extend(['-headers', ''.join(f'{key}: {val}\r\n' for key, val in fmt['http_headers'].items())])
@@ -572,7 +572,21 @@ class FFmpegFD(ExternalFD):
if end_time: if end_time:
args += ['-t', str(end_time - start_time)] args += ['-t', str(end_time - start_time)]
args += [*self._configuration_args((f'_i{i + 1}', '_i')), '-i', fmt['url']] url = fmt['url']
if self.params.get('enable_file_urls') and url.startswith('file:'):
# The default protocol_whitelist is 'file,crypto,data' when reading local m3u8 URLs,
# so only local segments can be read unless we also include 'http,https,tcp,tls'
args += ['-protocol_whitelist', 'file,crypto,data,http,https,tcp,tls']
# ffmpeg incorrectly handles 'file:' URLs by only removing the
# 'file:' prefix and treating the rest as if it's a normal filepath.
# FFmpegPostProcessor also depends on this behavior, so we need to fixup the URLs:
# - On Windows/Cygwin, replace 'file:///' and 'file://localhost/' with 'file:'
# - On *nix, replace 'file://localhost/' with 'file:/'
# Ref: https://github.com/yt-dlp/yt-dlp/issues/13781
# https://trac.ffmpeg.org/ticket/2702
url = re.sub(r'^file://(?:localhost)?/', 'file:' if os.name == 'nt' else 'file:/', url)
args += [*self._configuration_args((f'_i{i + 1}', '_i')), '-i', url]
if not (start_time or end_time) or not self.params.get('force_keyframes_at_cuts'): if not (start_time or end_time) or not self.params.get('force_keyframes_at_cuts'):
args += ['-c', 'copy'] args += ['-c', 'copy']
@@ -640,10 +654,6 @@ class FFmpegFD(ExternalFD):
return retval return retval
class AVconvFD(FFmpegFD):
pass
_BY_NAME = { _BY_NAME = {
klass.get_basename(): klass klass.get_basename(): klass
for name, klass in globals().items() for name, klass in globals().items()
plugins/youtube_download/yt_dlp/downloader/f4m.py
+5 -5
View File
@@ -149,14 +149,14 @@ class FlvReader(io.BytesIO):
segments_count = self.read_unsigned_char() segments_count = self.read_unsigned_char()
segments = [] segments = []
for _ in range(segments_count): for _ in range(segments_count):
box_size, box_type, box_data = self.read_box_info() _box_size, box_type, box_data = self.read_box_info()
assert box_type == b'asrt' assert box_type == b'asrt'
segment = FlvReader(box_data).read_asrt() segment = FlvReader(box_data).read_asrt()
segments.append(segment) segments.append(segment)
fragments_run_count = self.read_unsigned_char() fragments_run_count = self.read_unsigned_char()
fragments = [] fragments = []
for _ in range(fragments_run_count): for _ in range(fragments_run_count):
box_size, box_type, box_data = self.read_box_info() _box_size, box_type, box_data = self.read_box_info()
assert box_type == b'afrt' assert box_type == b'afrt'
fragments.append(FlvReader(box_data).read_afrt()) fragments.append(FlvReader(box_data).read_afrt())
@@ -167,7 +167,7 @@ class FlvReader(io.BytesIO):
} }
def read_bootstrap_info(self): def read_bootstrap_info(self):
total_size, box_type, box_data = self.read_box_info() _, box_type, box_data = self.read_box_info()
assert box_type == b'abst' assert box_type == b'abst'
return FlvReader(box_data).read_abst() return FlvReader(box_data).read_abst()
@@ -324,9 +324,9 @@ class F4mFD(FragmentFD):
if requested_bitrate is None or len(formats) == 1: if requested_bitrate is None or len(formats) == 1:
# get the best format # get the best format
formats = sorted(formats, key=lambda f: f[0]) formats = sorted(formats, key=lambda f: f[0])
rate, media = formats[-1] _, media = formats[-1]
else: else:
rate, media = next(filter( _, media = next(filter(
lambda f: int(f[0]) == requested_bitrate, formats)) lambda f: int(f[0]) == requested_bitrate, formats))
# Prefer baseURL for relative URLs as per 11.2 of F4M 3.0 spec. # Prefer baseURL for relative URLs as per 11.2 of F4M 3.0 spec.
plugins/youtube_download/yt_dlp/downloader/fragment.py
+1 -1
View File
@@ -302,7 +302,7 @@ class FragmentFD(FileDownloader):
elif to_file: elif to_file:
self.try_rename(ctx['tmpfilename'], ctx['filename']) self.try_rename(ctx['tmpfilename'], ctx['filename'])
filetime = ctx.get('fragment_filetime') filetime = ctx.get('fragment_filetime')
if self.params.get('updatetime', True) and filetime: if self.params.get('updatetime') and filetime:
with contextlib.suppress(Exception): with contextlib.suppress(Exception):
os.utime(ctx['filename'], (time.time(), filetime)) os.utime(ctx['filename'], (time.time(), filetime))
plugins/youtube_download/yt_dlp/downloader/hls.py
+15 -8
View File
@@ -94,12 +94,19 @@ class HlsFD(FragmentFD):
can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None
if can_download: if can_download:
has_ffmpeg = FFmpegFD.available() has_ffmpeg = FFmpegFD.available()
no_crypto = not Cryptodome.AES and '#EXT-X-KEY:METHOD=AES-128' in s if not Cryptodome.AES and '#EXT-X-KEY:METHOD=AES-128' in s:
if no_crypto and has_ffmpeg: # Even if pycryptodomex isn't available, force HlsFD for m3u8s that won't work with ffmpeg
can_download, message = False, 'The stream has AES-128 encryption and pycryptodomex is not available' ffmpeg_can_dl = not traverse_obj(info_dict, ((
elif no_crypto: 'extra_param_to_segment_url', 'extra_param_to_key_url',
message = ('The stream has AES-128 encryption and neither ffmpeg nor pycryptodomex are available; ' 'hls_media_playlist_data', ('hls_aes', ('uri', 'key', 'iv')),
'Decryption will be performed natively, but will be extremely slow') ), any))
message = 'The stream has AES-128 encryption and {} available'.format(
'neither ffmpeg nor pycryptodomex are' if ffmpeg_can_dl and not has_ffmpeg else
'pycryptodomex is not')
if has_ffmpeg and ffmpeg_can_dl:
can_download = False
else:
message += '; decryption will be performed natively, but will be extremely slow'
elif info_dict.get('extractor_key') == 'Generic' and re.search(r'(?m)#EXT-X-MEDIA-SEQUENCE:(?!0$)', s): elif info_dict.get('extractor_key') == 'Generic' and re.search(r'(?m)#EXT-X-MEDIA-SEQUENCE:(?!0$)', s):
install_ffmpeg = '' if has_ffmpeg else 'install ffmpeg and ' install_ffmpeg = '' if has_ffmpeg else 'install ffmpeg and '
message = ('Live HLS streams are not supported by the native downloader. If this is a livestream, ' message = ('Live HLS streams are not supported by the native downloader. If this is a livestream, '
@@ -198,7 +205,7 @@ class HlsFD(FragmentFD):
line = line.strip() line = line.strip()
if line: if line:
if not line.startswith('#'): if not line.startswith('#'):
if format_index and discontinuity_count != format_index: if format_index is not None and discontinuity_count != format_index:
continue continue
if ad_frag_next: if ad_frag_next:
continue continue
@@ -224,7 +231,7 @@ class HlsFD(FragmentFD):
byte_range = {} byte_range = {}
elif line.startswith('#EXT-X-MAP'): elif line.startswith('#EXT-X-MAP'):
if format_index and discontinuity_count != format_index: if format_index is not None and discontinuity_count != format_index:
continue continue
if frag_index > 0: if frag_index > 0:
self.report_error( self.report_error(
plugins/youtube_download/yt_dlp/downloader/http.py
+6 -11
View File
@@ -13,12 +13,9 @@ from ..utils import (
ContentTooShortError, ContentTooShortError,
RetryManager, RetryManager,
ThrottledDownload, ThrottledDownload,
XAttrMetadataError,
XAttrUnavailableError,
int_or_none, int_or_none,
parse_http_range, parse_http_range,
try_call, try_call,
write_xattr,
) )
from ..utils.networking import HTTPHeaderDict from ..utils.networking import HTTPHeaderDict
@@ -27,6 +24,10 @@ class HttpFD(FileDownloader):
def real_download(self, filename, info_dict): def real_download(self, filename, info_dict):
url = info_dict['url'] url = info_dict['url']
request_data = info_dict.get('request_data', None) request_data = info_dict.get('request_data', None)
request_extensions = {}
impersonate_target = self._get_impersonate_target(info_dict)
if impersonate_target is not None:
request_extensions['impersonate'] = impersonate_target
class DownloadContext(dict): class DownloadContext(dict):
__getattr__ = dict.get __getattr__ = dict.get
@@ -109,7 +110,7 @@ class HttpFD(FileDownloader):
if try_call(lambda: range_end >= ctx.content_len): if try_call(lambda: range_end >= ctx.content_len):
range_end = ctx.content_len - 1 range_end = ctx.content_len - 1
request = Request(url, request_data, headers) request = Request(url, request_data, headers, extensions=request_extensions)
has_range = range_start is not None has_range = range_start is not None
if has_range: if has_range:
request.headers['Range'] = f'bytes={int(range_start)}-{int_or_none(range_end) or ""}' request.headers['Range'] = f'bytes={int(range_start)}-{int_or_none(range_end) or ""}'
@@ -269,12 +270,6 @@ class HttpFD(FileDownloader):
self.report_error(f'unable to open for writing: {err}') self.report_error(f'unable to open for writing: {err}')
return False return False
if self.params.get('xattr_set_filesize', False) and data_len is not None:
try:
write_xattr(ctx.tmpfilename, 'user.ytdl.filesize', str(data_len).encode())
except (XAttrUnavailableError, XAttrMetadataError) as err:
self.report_error(f'unable to set filesize xattr: {err}')
try: try:
ctx.stream.write(data_block) ctx.stream.write(data_block)
except OSError as err: except OSError as err:
@@ -348,7 +343,7 @@ class HttpFD(FileDownloader):
self.try_rename(ctx.tmpfilename, ctx.filename) self.try_rename(ctx.tmpfilename, ctx.filename)
# Update file modification time # Update file modification time
if self.params.get('updatetime', True): if self.params.get('updatetime'):
info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.headers.get('last-modified', None)) info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.headers.get('last-modified', None))
self._hook_progress({ self._hook_progress({
plugins/youtube_download/yt_dlp/downloader/niconico.py
+25 -28
View File
@@ -5,47 +5,46 @@ import time
from .common import FileDownloader from .common import FileDownloader
from .external import FFmpegFD from .external import FFmpegFD
from ..networking import Request from ..networking import Request
from ..utils import DownloadError, str_or_none, try_get from ..networking.websocket import WebSocketResponse
from ..utils import DownloadError, str_or_none, truncate_string
from ..utils.traversal import traverse_obj
class NiconicoLiveFD(FileDownloader): class NiconicoLiveFD(FileDownloader):
""" Downloads niconico live without being stopped """ """ Downloads niconico live without being stopped """
def real_download(self, filename, info_dict): def real_download(self, filename, info_dict):
video_id = info_dict['video_id'] video_id = info_dict['id']
ws_url = info_dict['url'] opts = info_dict['downloader_options']
ws_extractor = info_dict['ws'] quality, ws_extractor, ws_url = opts['max_quality'], opts['ws'], opts['ws_url']
ws_origin_host = info_dict['origin']
live_quality = info_dict.get('live_quality', 'high')
live_latency = info_dict.get('live_latency', 'high')
dl = FFmpegFD(self.ydl, self.params or {}) dl = FFmpegFD(self.ydl, self.params or {})
new_info_dict = info_dict.copy() new_info_dict = info_dict.copy()
new_info_dict.update({ new_info_dict['protocol'] = 'm3u8'
'protocol': 'm3u8',
})
def communicate_ws(reconnect): def communicate_ws(reconnect):
if reconnect: # Support --load-info-json as if it is a reconnect attempt
ws = self.ydl.urlopen(Request(ws_url, headers={'Origin': f'https://{ws_origin_host}'})) if reconnect or not isinstance(ws_extractor, WebSocketResponse):
ws = self.ydl.urlopen(Request(
ws_url, headers={'Origin': 'https://live.nicovideo.jp'}))
if self.ydl.params.get('verbose', False): if self.ydl.params.get('verbose', False):
self.to_screen('[debug] Sending startWatching request') self.write_debug('Sending startWatching request')
ws.send(json.dumps({ ws.send(json.dumps({
'type': 'startWatching',
'data': { 'data': {
'reconnect': True,
'room': {
'commentable': True,
'protocol': 'webSocket',
},
'stream': { 'stream': {
'quality': live_quality,
'protocol': 'hls+fmp4',
'latency': live_latency,
'accessRightMethod': 'single_cookie', 'accessRightMethod': 'single_cookie',
'chasePlay': False, 'chasePlay': False,
'latency': 'high',
'protocol': 'hls',
'quality': quality,
}, },
'room': {
'protocol': 'webSocket',
'commentable': True,
},
'reconnect': True,
}, },
'type': 'startWatching',
})) }))
else: else:
ws = ws_extractor ws = ws_extractor
@@ -58,7 +57,6 @@ class NiconicoLiveFD(FileDownloader):
if not data or not isinstance(data, dict): if not data or not isinstance(data, dict):
continue continue
if data.get('type') == 'ping': if data.get('type') == 'ping':
# pong back
ws.send(r'{"type":"pong"}') ws.send(r'{"type":"pong"}')
ws.send(r'{"type":"keepSeat"}') ws.send(r'{"type":"keepSeat"}')
elif data.get('type') == 'disconnect': elif data.get('type') == 'disconnect':
@@ -66,12 +64,10 @@ class NiconicoLiveFD(FileDownloader):
return True return True
elif data.get('type') == 'error': elif data.get('type') == 'error':
self.write_debug(data) self.write_debug(data)
message = try_get(data, lambda x: x['body']['code'], str) or recv message = traverse_obj(data, ('body', 'code', {str_or_none}), default=recv)
return DownloadError(message) return DownloadError(message)
elif self.ydl.params.get('verbose', False): elif self.ydl.params.get('verbose', False):
if len(recv) > 100: self.write_debug(f'Server response: {truncate_string(recv, 100)}')
recv = recv[:100] + '...'
self.to_screen(f'[debug] Server said: {recv}')
def ws_main(): def ws_main():
reconnect = False reconnect = False
@@ -81,7 +77,8 @@ class NiconicoLiveFD(FileDownloader):
if ret is True: if ret is True:
return return
except BaseException as e: except BaseException as e:
self.to_screen('[{}] {}: Connection error occured, reconnecting after 10 seconds: {}'.format('niconico:live', video_id, str_or_none(e))) self.to_screen(
f'[niconico:live] {video_id}: Connection error occured, reconnecting after 10 seconds: {e}')
time.sleep(10) time.sleep(10)
continue continue
finally: finally:
plugins/youtube_download/yt_dlp/extractor/_extractors.py
+53 -85
View File
@@ -58,13 +58,7 @@ from .adn import (
ADNSeasonIE, ADNSeasonIE,
) )
from .adobeconnect import AdobeConnectIE from .adobeconnect import AdobeConnectIE
from .adobetv import ( from .adobetv import AdobeTVVideoIE
AdobeTVChannelIE,
AdobeTVEmbedIE,
AdobeTVIE,
AdobeTVShowIE,
AdobeTVVideoIE,
)
from .adultswim import AdultSwimIE from .adultswim import AdultSwimIE
from .aenetworks import ( from .aenetworks import (
AENetworksCollectionIE, AENetworksCollectionIE,
@@ -152,7 +146,6 @@ from .ard import (
ARDBetaMediathekIE, ARDBetaMediathekIE,
ARDMediathekCollectionIE, ARDMediathekCollectionIE,
) )
from .arkena import ArkenaIE
from .arnes import ArnesIE from .arnes import ArnesIE
from .art19 import ( from .art19 import (
Art19IE, Art19IE,
@@ -201,7 +194,6 @@ from .banbye import (
BanByeChannelIE, BanByeChannelIE,
BanByeIE, BanByeIE,
) )
from .bandaichannel import BandaiChannelIE
from .bandcamp import ( from .bandcamp import (
BandcampAlbumIE, BandcampAlbumIE,
BandcampIE, BandcampIE,
@@ -229,7 +221,6 @@ from .beatbump import (
from .beatport import BeatportIE from .beatport import BeatportIE
from .beeg import BeegIE from .beeg import BeegIE
from .behindkink import BehindKinkIE from .behindkink import BehindKinkIE
from .bellmedia import BellMediaIE
from .berufetv import BerufeTVIE from .berufetv import BerufeTVIE
from .bet import BetIE from .bet import BetIE
from .bfi import BFIPlayerIE from .bfi import BFIPlayerIE
@@ -275,7 +266,10 @@ from .bitchute import (
BitChuteChannelIE, BitChuteChannelIE,
BitChuteIE, BitChuteIE,
) )
from .blackboardcollaborate import BlackboardCollaborateIE from .blackboardcollaborate import (
BlackboardCollaborateIE,
BlackboardCollaborateLaunchIE,
)
from .bleacherreport import ( from .bleacherreport import (
BleacherReportCMSIE, BleacherReportCMSIE,
BleacherReportIE, BleacherReportIE,
@@ -300,7 +294,6 @@ from .brainpop import (
BrainPOPIlIE, BrainPOPIlIE,
BrainPOPJrIE, BrainPOPJrIE,
) )
from .bravotv import BravoTVIE
from .breitbart import BreitBartIE from .breitbart import BreitBartIE
from .brightcove import ( from .brightcove import (
BrightcoveLegacyIE, BrightcoveLegacyIE,
@@ -310,6 +303,7 @@ from .brilliantpala import (
BrilliantpalaClassesIE, BrilliantpalaClassesIE,
BrilliantpalaElearnIE, BrilliantpalaElearnIE,
) )
from .btvplus import BTVPlusIE
from .bundesliga import BundesligaIE from .bundesliga import BundesligaIE
from .bundestag import BundestagIE from .bundestag import BundestagIE
from .bunnycdn import BunnyCdnIE from .bunnycdn import BunnyCdnIE
@@ -338,12 +332,12 @@ from .canalc2 import Canalc2IE
from .canalplus import CanalplusIE from .canalplus import CanalplusIE
from .canalsurmas import CanalsurmasIE from .canalsurmas import CanalsurmasIE
from .caracoltv import CaracolTvPlayIE from .caracoltv import CaracolTvPlayIE
from .cartoonnetwork import CartoonNetworkIE
from .cbc import ( from .cbc import (
CBCIE, CBCIE,
CBCGemIE, CBCGemIE,
CBCGemLiveIE, CBCGemLiveIE,
CBCGemPlaylistIE, CBCGemPlaylistIE,
CBCListenIE,
CBCPlayerIE, CBCPlayerIE,
CBCPlayerPlaylistIE, CBCPlayerPlaylistIE,
) )
@@ -405,16 +399,12 @@ from .cloudflarestream import CloudflareStreamIE
from .cloudycdn import CloudyCDNIE from .cloudycdn import CloudyCDNIE
from .clubic import ClubicIE from .clubic import ClubicIE
from .clyp import ClypIE from .clyp import ClypIE
from .cmt import CMTIE
from .cnbc import CNBCVideoIE from .cnbc import CNBCVideoIE
from .cnn import ( from .cnn import (
CNNIE, CNNIE,
CNNIndonesiaIE, CNNIndonesiaIE,
) )
from .comedycentral import ( from .comedycentral import ComedyCentralIE
ComedyCentralIE,
ComedyCentralTVIE,
)
from .commonmistakes import ( from .commonmistakes import (
BlobIE, BlobIE,
CommonMistakesIE, CommonMistakesIE,
@@ -435,7 +425,6 @@ from .cpac import (
CPACPlaylistIE, CPACPlaylistIE,
) )
from .cracked import CrackedIE from .cracked import CrackedIE
from .crackle import CrackleIE
from .craftsy import CraftsyIE from .craftsy import CraftsyIE
from .crooksandliars import CrooksAndLiarsIE from .crooksandliars import CrooksAndLiarsIE
from .crowdbunker import ( from .crowdbunker import (
@@ -448,7 +437,6 @@ from .cspan import (
CSpanIE, CSpanIE,
) )
from .ctsnews import CtsNewsIE from .ctsnews import CtsNewsIE
from .ctv import CTVIE
from .ctvnews import CTVNewsIE from .ctvnews import CTVNewsIE
from .cultureunplugged import CultureUnpluggedIE from .cultureunplugged import CultureUnpluggedIE
from .curiositystream import ( from .curiositystream import (
@@ -456,10 +444,6 @@ from .curiositystream import (
CuriosityStreamIE, CuriosityStreamIE,
CuriosityStreamSeriesIE, CuriosityStreamSeriesIE,
) )
from .cwtv import (
CWTVIE,
CWTVMovieIE,
)
from .cybrary import ( from .cybrary import (
CybraryCourseIE, CybraryCourseIE,
CybraryIE, CybraryIE,
@@ -572,10 +556,6 @@ from .dw import (
DWIE, DWIE,
DWArticleIE, DWArticleIE,
) )
from .eagleplatform import (
ClipYouEmbedIE,
EaglePlatformIE,
)
from .ebaumsworld import EbaumsWorldIE from .ebaumsworld import EbaumsWorldIE
from .ebay import EbayIE from .ebay import EbayIE
from .egghead import ( from .egghead import (
@@ -641,6 +621,10 @@ from .fancode import (
FancodeVodIE, FancodeVodIE,
) )
from .fathom import FathomIE from .fathom import FathomIE
from .faulio import (
FaulioIE,
FaulioLiveIE,
)
from .faz import FazIE from .faz import FazIE
from .fc2 import ( from .fc2 import (
FC2IE, FC2IE,
@@ -807,9 +791,7 @@ from .holodex import HolodexIE
from .hotnewhiphop import HotNewHipHopIE from .hotnewhiphop import HotNewHipHopIE
from .hotstar import ( from .hotstar import (
HotStarIE, HotStarIE,
HotStarPlaylistIE,
HotStarPrefixIE, HotStarPrefixIE,
HotStarSeasonIE,
HotStarSeriesIE, HotStarSeriesIE,
) )
from .hrefli import HrefLiRedirectIE from .hrefli import HrefLiRedirectIE
@@ -842,6 +824,13 @@ from .ichinanalive import (
IchinanaLiveIE, IchinanaLiveIE,
IchinanaLiveVODIE, IchinanaLiveVODIE,
) )
from .idagio import (
IdagioAlbumIE,
IdagioPersonalPlaylistIE,
IdagioPlaylistIE,
IdagioRecordingIE,
IdagioTrackIE,
)
from .idolplus import IdolPlusIE from .idolplus import IdolPlusIE
from .ign import ( from .ign import (
IGNIE, IGNIE,
@@ -923,17 +912,15 @@ from .japandiet import (
ShugiinItvVodIE, ShugiinItvVodIE,
) )
from .jeuxvideo import JeuxVideoIE from .jeuxvideo import JeuxVideoIE
from .jiocinema import (
JioCinemaIE,
JioCinemaSeriesIE,
)
from .jiosaavn import ( from .jiosaavn import (
JioSaavnAlbumIE, JioSaavnAlbumIE,
JioSaavnArtistIE,
JioSaavnPlaylistIE, JioSaavnPlaylistIE,
JioSaavnShowIE,
JioSaavnShowPlaylistIE,
JioSaavnSongIE, JioSaavnSongIE,
) )
from .joj import JojIE from .joj import JojIE
from .joqrag import JoqrAgIE
from .jove import JoveIE from .jove import JoveIE
from .jstream import JStreamIE from .jstream import JStreamIE
from .jtbc import ( from .jtbc import (
@@ -1036,11 +1023,6 @@ from .likee import (
LikeeIE, LikeeIE,
LikeeUserIE, LikeeUserIE,
) )
from .limelight import (
LimelightChannelIE,
LimelightChannelListIE,
LimelightMediaIE,
)
from .linkedin import ( from .linkedin import (
LinkedInEventsIE, LinkedInEventsIE,
LinkedInIE, LinkedInIE,
@@ -1106,6 +1088,7 @@ from .markiza import (
from .massengeschmacktv import MassengeschmackTVIE from .massengeschmacktv import MassengeschmackTVIE
from .masters import MastersIE from .masters import MastersIE
from .matchtv import MatchTVIE from .matchtv import MatchTVIE
from .mave import MaveIE
from .mbn import MBNIE from .mbn import MBNIE
from .mdr import MDRIE from .mdr import MDRIE
from .medaltv import MedalTVIE from .medaltv import MedalTVIE
@@ -1151,6 +1134,7 @@ from .minds import (
MindsIE, MindsIE,
) )
from .minoto import MinotoIE from .minoto import MinotoIE
from .mir24tv import Mir24TvIE
from .mirrativ import ( from .mirrativ import (
MirrativIE, MirrativIE,
MirrativUserIE, MirrativUserIE,
@@ -1160,7 +1144,6 @@ from .mit import (
OCWMITIE, OCWMITIE,
TechTVMITIE, TechTVMITIE,
) )
from .mitele import MiTeleIE
from .mixch import ( from .mixch import (
MixchArchiveIE, MixchArchiveIE,
MixchIE, MixchIE,
@@ -1171,6 +1154,10 @@ from .mixcloud import (
MixcloudPlaylistIE, MixcloudPlaylistIE,
MixcloudUserIE, MixcloudUserIE,
) )
from .mixlr import (
MixlrIE,
MixlrRecoringIE,
)
from .mlb import ( from .mlb import (
MLBIE, MLBIE,
MLBTVIE, MLBTVIE,
@@ -1194,15 +1181,7 @@ from .moview import MoviewPlayIE
from .moviezine import MoviezineIE from .moviezine import MoviezineIE
from .movingimage import MovingImageIE from .movingimage import MovingImageIE
from .msn import MSNIE from .msn import MSNIE
from .mtv import ( from .mtv import MTVIE
MTVDEIE,
MTVIE,
MTVItaliaIE,
MTVItaliaProgrammaIE,
MTVJapanIE,
MTVServicesEmbeddedIE,
MTVVideoIE,
)
from .muenchentv import MuenchenTVIE from .muenchentv import MuenchenTVIE
from .murrtube import ( from .murrtube import (
MurrtubeIE, MurrtubeIE,
@@ -1260,6 +1239,7 @@ from .nba import (
) )
from .nbc import ( from .nbc import (
NBCIE, NBCIE,
BravoTVIE,
NBCNewsIE, NBCNewsIE,
NBCOlympicsIE, NBCOlympicsIE,
NBCOlympicsStreamIE, NBCOlympicsStreamIE,
@@ -1267,6 +1247,7 @@ from .nbc import (
NBCSportsStreamIE, NBCSportsStreamIE,
NBCSportsVPlayerIE, NBCSportsVPlayerIE,
NBCStationsIE, NBCStationsIE,
SyfyIE,
) )
from .ndr import ( from .ndr import (
NDRIE, NDRIE,
@@ -1342,12 +1323,7 @@ from .nhk import (
NhkVodProgramIE, NhkVodProgramIE,
) )
from .nhl import NHLIE from .nhl import NHLIE
from .nick import ( from .nick import NickIE
NickBrIE,
NickDeIE,
NickIE,
NickRuIE,
)
from .niconico import ( from .niconico import (
NiconicoHistoryIE, NiconicoHistoryIE,
NiconicoIE, NiconicoIE,
@@ -1379,7 +1355,6 @@ from .nobelprize import NobelPrizeIE
from .noice import NoicePodcastIE from .noice import NoicePodcastIE
from .nonktube import NonkTubeIE from .nonktube import NonkTubeIE
from .noodlemagazine import NoodleMagazineIE from .noodlemagazine import NoodleMagazineIE
from .noovo import NoovoIE
from .nosnl import NOSNLArticleIE from .nosnl import NOSNLArticleIE
from .nova import ( from .nova import (
NovaEmbedIE, NovaEmbedIE,
@@ -1460,6 +1435,7 @@ from .onet import (
OnetPlIE, OnetPlIE,
) )
from .onionstudios import OnionStudiosIE from .onionstudios import OnionStudiosIE
from .onsen import OnsenIE
from .opencast import ( from .opencast import (
OpencastIE, OpencastIE,
OpencastPlaylistIE, OpencastPlaylistIE,
@@ -1493,10 +1469,6 @@ from .panopto import (
PanoptoListIE, PanoptoListIE,
PanoptoPlaylistIE, PanoptoPlaylistIE,
) )
from .paramountplus import (
ParamountPlusIE,
ParamountPlusSeriesIE,
)
from .parler import ParlerIE from .parler import ParlerIE
from .parlview import ParlviewIE from .parlview import ParlviewIE
from .parti import ( from .parti import (
@@ -1550,16 +1522,12 @@ from .piramidetv import (
PiramideTVChannelIE, PiramideTVChannelIE,
PiramideTVIE, PiramideTVIE,
) )
from .pixivsketch import (
PixivSketchIE,
PixivSketchUserIE,
)
from .pladform import PladformIE
from .planetmarathi import PlanetMarathiIE from .planetmarathi import PlanetMarathiIE
from .platzi import ( from .platzi import (
PlatziCourseIE, PlatziCourseIE,
PlatziIE, PlatziIE,
) )
from .playerfm import PlayerFmIE
from .playplustv import PlayPlusTVIE from .playplustv import PlayPlusTVIE
from .playsuisse import PlaySuisseIE from .playsuisse import PlaySuisseIE
from .playtvak import PlaytvakIE from .playtvak import PlaytvakIE
@@ -1570,6 +1538,7 @@ from .pluralsight import (
) )
from .plutotv import PlutoTVIE from .plutotv import PlutoTVIE
from .plvideo import PlVideoIE from .plvideo import PlVideoIE
from .plyr import PlyrEmbedIE
from .podbayfm import ( from .podbayfm import (
PodbayFMChannelIE, PodbayFMChannelIE,
PodbayFMIE, PodbayFMIE,
@@ -1785,6 +1754,7 @@ from .rtve import (
RTVEALaCartaIE, RTVEALaCartaIE,
RTVEAudioIE, RTVEAudioIE,
RTVELiveIE, RTVELiveIE,
RTVEProgramIE,
RTVETelevisionIE, RTVETelevisionIE,
) )
from .rtvs import RTVSIE from .rtvs import RTVSIE
@@ -1808,7 +1778,6 @@ from .rutube import (
RutubePlaylistIE, RutubePlaylistIE,
RutubeTagsIE, RutubeTagsIE,
) )
from .rutv import RUTVIE
from .ruutu import RuutuIE from .ruutu import RuutuIE
from .ruv import ( from .ruv import (
RuvIE, RuvIE,
@@ -1826,6 +1795,7 @@ from .safari import (
from .saitosan import SaitosanIE from .saitosan import SaitosanIE
from .samplefocus import SampleFocusIE from .samplefocus import SampleFocusIE
from .sapo import SapoIE from .sapo import SapoIE
from .sauceplus import SaucePlusIE
from .sbs import SBSIE from .sbs import SBSIE
from .sbscokr import ( from .sbscokr import (
SBSCoKrAllvodProgramIE, SBSCoKrAllvodProgramIE,
@@ -1868,6 +1838,7 @@ from .shahid import (
from .sharepoint import SharePointIE from .sharepoint import SharePointIE
from .sharevideos import ShareVideosEmbedIE from .sharevideos import ShareVideosEmbedIE
from .shemaroome import ShemarooMeIE from .shemaroome import ShemarooMeIE
from .shiey import ShieyIE
from .showroomlive import ShowRoomLiveIE from .showroomlive import ShowRoomLiveIE
from .sibnet import SibnetEmbedIE from .sibnet import SibnetEmbedIE
from .simplecast import ( from .simplecast import (
@@ -1876,7 +1847,6 @@ from .simplecast import (
SimplecastPodcastIE, SimplecastPodcastIE,
) )
from .sina import SinaIE from .sina import SinaIE
from .sixplay import SixPlayIE
from .skeb import SkebIE from .skeb import SkebIE
from .sky import ( from .sky import (
SkyNewsIE, SkyNewsIE,
@@ -1904,7 +1874,12 @@ from .skynewsau import SkyNewsAUIE
from .slideshare import SlideshareIE from .slideshare import SlideshareIE
from .slideslive import SlidesLiveIE from .slideslive import SlidesLiveIE
from .slutload import SlutloadIE from .slutload import SlutloadIE
from .smotrim import SmotrimIE from .smotrim import (
SmotrimAudioIE,
SmotrimIE,
SmotrimLiveIE,
SmotrimPlaylistIE,
)
from .snapchat import SnapchatSpotlightIE from .snapchat import SnapchatSpotlightIE
from .snotr import SnotrIE from .snotr import SnotrIE
from .softwhiteunderbelly import SoftWhiteUnderbellyIE from .softwhiteunderbelly import SoftWhiteUnderbellyIE
@@ -1932,12 +1907,13 @@ from .soundgasm import (
SoundgasmProfileIE, SoundgasmProfileIE,
) )
from .southpark import ( from .southpark import (
SouthParkComBrIE,
SouthParkCoUkIE,
SouthParkDeIE, SouthParkDeIE,
SouthParkDkIE, SouthParkDkIE,
SouthParkEsIE, SouthParkEsIE,
SouthParkIE, SouthParkIE,
SouthParkLatIE, SouthParkLatIE,
SouthParkNlIE,
) )
from .sovietscloset import ( from .sovietscloset import (
SovietsClosetIE, SovietsClosetIE,
@@ -1948,23 +1924,14 @@ from .spankbang import (
SpankBangPlaylistIE, SpankBangPlaylistIE,
) )
from .spiegel import SpiegelIE from .spiegel import SpiegelIE
from .spike import (
BellatorIE,
ParamountNetworkIE,
)
from .sport5 import Sport5IE from .sport5 import Sport5IE
from .sportbox import SportBoxIE from .sportbox import SportBoxIE
from .sportdeutschland import SportDeutschlandIE from .sportdeutschland import SportDeutschlandIE
from .spotify import (
SpotifyIE,
SpotifyShowIE,
)
from .spreaker import ( from .spreaker import (
SpreakerIE, SpreakerIE,
SpreakerShowIE, SpreakerShowIE,
) )
from .springboardplatform import SpringboardPlatformIE from .springboardplatform import SpringboardPlatformIE
from .sprout import SproutIE
from .sproutvideo import ( from .sproutvideo import (
SproutVideoIE, SproutVideoIE,
VidsIoIE, VidsIoIE,
@@ -1986,6 +1953,7 @@ from .startrek import StarTrekIE
from .startv import StarTVIE from .startv import StarTVIE
from .steam import ( from .steam import (
SteamCommunityBroadcastIE, SteamCommunityBroadcastIE,
SteamCommunityIE,
SteamIE, SteamIE,
) )
from .stitcher import ( from .stitcher import (
@@ -2015,13 +1983,11 @@ from .sverigesradio import (
SverigesRadioPublicationIE, SverigesRadioPublicationIE,
) )
from .svt import ( from .svt import (
SVTIE,
SVTPageIE, SVTPageIE,
SVTPlayIE, SVTPlayIE,
SVTSeriesIE, SVTSeriesIE,
) )
from .swearnet import SwearnetEpisodeIE from .swearnet import SwearnetEpisodeIE
from .syfy import SyfyIE
from .syvdk import SYVDKIE from .syvdk import SYVDKIE
from .sztvhu import SztvHuIE from .sztvhu import SztvHuIE
from .tagesschau import TagesschauIE from .tagesschau import TagesschauIE
@@ -2100,6 +2066,7 @@ from .theguardian import (
TheGuardianPodcastIE, TheGuardianPodcastIE,
TheGuardianPodcastPlaylistIE, TheGuardianPodcastPlaylistIE,
) )
from .thehighwire import TheHighWireIE
from .theholetv import TheHoleTvIE from .theholetv import TheHoleTvIE
from .theintercept import TheInterceptIE from .theintercept import TheInterceptIE
from .theplatform import ( from .theplatform import (
@@ -2146,6 +2113,7 @@ from .toggle import (
from .toggo import ToggoIE from .toggo import ToggoIE
from .tonline import TOnlineIE from .tonline import TOnlineIE
from .toongoggles import ToonGogglesIE from .toongoggles import ToonGogglesIE
from .toutiao import ToutiaoIE
from .toutv import TouTvIE from .toutv import TouTvIE
from .toypics import ( from .toypics import (
ToypicsIE, ToypicsIE,
@@ -2168,7 +2136,6 @@ from .trtworld import TrtWorldIE
from .trueid import TrueIDIE from .trueid import TrueIDIE
from .trunews import TruNewsIE from .trunews import TruNewsIE
from .truth import TruthIE from .truth import TruthIE
from .trutv import TruTVIE
from .tube8 import Tube8IE from .tube8 import Tube8IE
from .tubetugraz import ( from .tubetugraz import (
TubeTuGrazIE, TubeTuGrazIE,
@@ -2180,6 +2147,7 @@ from .tubitv import (
) )
from .tumblr import TumblrIE from .tumblr import TumblrIE
from .tunein import ( from .tunein import (
TuneInEmbedIE,
TuneInPodcastEpisodeIE, TuneInPodcastEpisodeIE,
TuneInPodcastIE, TuneInPodcastIE,
TuneInShortenerIE, TuneInShortenerIE,
@@ -2218,7 +2186,6 @@ from .tvc import (
from .tver import TVerIE from .tver import TVerIE
from .tvigle import TvigleIE from .tvigle import TvigleIE
from .tviplayer import TVIPlayerIE from .tviplayer import TVIPlayerIE
from .tvland import TVLandIE
from .tvn24 import TVN24IE from .tvn24 import TVN24IE
from .tvnoe import TVNoeIE from .tvnoe import TVNoeIE
from .tvopengr import ( from .tvopengr import (
@@ -2239,6 +2206,7 @@ from .tvplay import (
from .tvplayer import TVPlayerIE from .tvplayer import TVPlayerIE
from .tvw import ( from .tvw import (
TvwIE, TvwIE,
TvwNewsIE,
TvwTvChannelsIE, TvwTvChannelsIE,
) )
from .tweakers import TweakersIE from .tweakers import TweakersIE
@@ -2287,6 +2255,7 @@ from .uliza import (
) )
from .umg import UMGDeIE from .umg import UMGDeIE
from .unistra import UnistraIE from .unistra import UnistraIE
from .unitednations import UnitedNationsWebTvIE
from .unity import UnityIE from .unity import UnityIE
from .unsupported import ( from .unsupported import (
KnownDRMIE, KnownDRMIE,
@@ -2313,7 +2282,6 @@ from .utreon import UtreonIE
from .varzesh3 import Varzesh3IE from .varzesh3 import Varzesh3IE
from .vbox7 import Vbox7IE from .vbox7 import Vbox7IE
from .veo import VeoIE from .veo import VeoIE
from .vesti import VestiIE
from .vevo import ( from .vevo import (
VevoIE, VevoIE,
VevoPlaylistIE, VevoPlaylistIE,
@@ -2368,6 +2336,7 @@ from .vimeo import (
VHXEmbedIE, VHXEmbedIE,
VimeoAlbumIE, VimeoAlbumIE,
VimeoChannelIE, VimeoChannelIE,
VimeoEventIE,
VimeoGroupsIE, VimeoGroupsIE,
VimeoIE, VimeoIE,
VimeoLikesIE, VimeoLikesIE,
@@ -2501,7 +2470,6 @@ from .wykop import (
WykopPostCommentIE, WykopPostCommentIE,
WykopPostIE, WykopPostIE,
) )
from .xanimu import XanimuIE
from .xboxclips import XboxClipsIE from .xboxclips import XboxClipsIE
from .xhamster import ( from .xhamster import (
XHamsterEmbedIE, XHamsterEmbedIE,
plugins/youtube_download/yt_dlp/extractor/abc.py
+18 -6
View File
@@ -21,7 +21,7 @@ from ..utils import (
class ABCIE(InfoExtractor): class ABCIE(InfoExtractor):
IE_NAME = 'abc.net.au' IE_NAME = 'abc.net.au'
_VALID_URL = r'https?://(?:www\.)?abc\.net\.au/(?:news|btn)/(?:[^/]+/){1,4}(?P<id>\d{5,})' _VALID_URL = r'https?://(?:www\.)?abc\.net\.au/(?:news|btn|listen)/(?:[^/?#]+/){1,4}(?P<id>\d{5,})'
_TESTS = [{ _TESTS = [{
'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334', 'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334',
@@ -53,8 +53,9 @@ class ABCIE(InfoExtractor):
'info_dict': { 'info_dict': {
'id': '6880080', 'id': '6880080',
'ext': 'mp3', 'ext': 'mp3',
'title': 'NAB lifts interest rates, following Westpac and CBA', 'title': 'NAB lifts interest rates, following Westpac and CBA - ABC listen',
'description': 'md5:f13d8edc81e462fce4a0437c7dc04728', 'description': 'md5:f13d8edc81e462fce4a0437c7dc04728',
'thumbnail': r're:https://live-production\.wcms\.abc-cdn\.net\.au/2193d7437c84b25eafd6360c82b5fa21',
}, },
}, { }, {
'url': 'http://www.abc.net.au/news/2015-10-19/6866214', 'url': 'http://www.abc.net.au/news/2015-10-19/6866214',
@@ -64,8 +65,9 @@ class ABCIE(InfoExtractor):
'info_dict': { 'info_dict': {
'id': '10527914', 'id': '10527914',
'ext': 'mp4', 'ext': 'mp4',
'title': 'WWI Centenary', 'title': 'WWI Centenary - Behind The News',
'description': 'md5:c2379ec0ca84072e86b446e536954546', 'description': 'md5:fa4405939ff750fade46ff0cd4c66a52',
'thumbnail': r're:https://live-production\.wcms\.abc-cdn\.net\.au/bcc3433c97bf992dff32ec5a768713c9',
}, },
}, { }, {
'url': 'https://www.abc.net.au/news/programs/the-world/2020-06-10/black-lives-matter-protests-spawn-support-for/12342074', 'url': 'https://www.abc.net.au/news/programs/the-world/2020-06-10/black-lives-matter-protests-spawn-support-for/12342074',
@@ -73,7 +75,8 @@ class ABCIE(InfoExtractor):
'id': '12342074', 'id': '12342074',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Black Lives Matter protests spawn support for Papuans in Indonesia', 'title': 'Black Lives Matter protests spawn support for Papuans in Indonesia',
'description': 'md5:2961a17dc53abc558589ccd0fb8edd6f', 'description': 'md5:625257209f2d14ce23cb4e3785da9beb',
'thumbnail': r're:https://live-production\.wcms\.abc-cdn\.net\.au/7ee6f190de6d7dbb04203e514bfae9ec',
}, },
}, { }, {
'url': 'https://www.abc.net.au/btn/newsbreak/btn-newsbreak-20200814/12560476', 'url': 'https://www.abc.net.au/btn/newsbreak/btn-newsbreak-20200814/12560476',
@@ -93,7 +96,16 @@ class ABCIE(InfoExtractor):
'title': 'Wagner Group retreating from Russia, leader Prigozhin to move to Belarus', 'title': 'Wagner Group retreating from Russia, leader Prigozhin to move to Belarus',
'ext': 'mp4', 'ext': 'mp4',
'description': 'Wagner troops leave Rostov-on-Don and\xa0Yevgeny Prigozhin will move to Belarus under a deal brokered by Belarusian President Alexander Lukashenko to end the mutiny.', 'description': 'Wagner troops leave Rostov-on-Don and\xa0Yevgeny Prigozhin will move to Belarus under a deal brokered by Belarusian President Alexander Lukashenko to end the mutiny.',
'thumbnail': 'https://live-production.wcms.abc-cdn.net.au/0c170f5b57f0105c432f366c0e8e267b?impolicy=wcms_crop_resize&cropH=2813&cropW=5000&xPos=0&yPos=249&width=862&height=485', 'thumbnail': r're:https://live-production\.wcm\.abc-cdn\.net\.au/0c170f5b57f0105c432f366c0e8e267b',
},
}, {
'url': 'https://www.abc.net.au/listen/programs/the-followers-madness-of-two/presents-followers-madness-of-two/105697646',
'info_dict': {
'id': '105697646',
'title': 'INTRODUCING — The Followers: Madness of Two - ABC listen',
'ext': 'mp3',
'description': 'md5:2310cd0d440a4e01656abea15db8d1f3',
'thumbnail': r're:https://live-production\.wcms\.abc-cdn\.net\.au/90d7078214e5d66553ffb7fcf0da0cda',
}, },
}] }]
plugins/youtube_download/yt_dlp/extractor/adobepass.py
+143 -48
View File
@@ -3,6 +3,7 @@ import json
import re import re
import time import time
import urllib.parse import urllib.parse
import uuid
import xml.etree.ElementTree as etree import xml.etree.ElementTree as etree
from .common import InfoExtractor from .common import InfoExtractor
@@ -10,6 +11,7 @@ from ..networking.exceptions import HTTPError
from ..utils import ( from ..utils import (
NO_DEFAULT, NO_DEFAULT,
ExtractorError, ExtractorError,
parse_qs,
unescapeHTML, unescapeHTML,
unified_timestamp, unified_timestamp,
urlencode_postdata, urlencode_postdata,
@@ -45,6 +47,7 @@ MSO_INFO = {
'name': 'Comcast XFINITY', 'name': 'Comcast XFINITY',
'username_field': 'user', 'username_field': 'user',
'password_field': 'passwd', 'password_field': 'passwd',
'login_hostname': 'login.xfinity.com',
}, },
'TWC': { 'TWC': {
'name': 'Time Warner Cable | Spectrum', 'name': 'Time Warner Cable | Spectrum',
@@ -74,6 +77,12 @@ MSO_INFO = {
'name': 'Verizon FiOS', 'name': 'Verizon FiOS',
'username_field': 'IDToken1', 'username_field': 'IDToken1',
'password_field': 'IDToken2', 'password_field': 'IDToken2',
'login_hostname': 'ssoauth.verizon.com',
},
'Fubo': {
'name': 'Fubo',
'username_field': 'username',
'password_field': 'password',
}, },
'Cablevision': { 'Cablevision': {
'name': 'Optimum/Cablevision', 'name': 'Optimum/Cablevision',
@@ -1338,6 +1347,7 @@ MSO_INFO = {
'name': 'Sling TV', 'name': 'Sling TV',
'username_field': 'username', 'username_field': 'username',
'password_field': 'password', 'password_field': 'password',
'login_hostname': 'identity.sling.com',
}, },
'Suddenlink': { 'Suddenlink': {
'name': 'Suddenlink', 'name': 'Suddenlink',
@@ -1355,7 +1365,6 @@ MSO_INFO = {
class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should end with BaseIE/InfoExtractor class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should end with BaseIE/InfoExtractor
_SERVICE_PROVIDER_TEMPLATE = 'https://sp.auth.adobe.com/adobe-services/%s' _SERVICE_PROVIDER_TEMPLATE = 'https://sp.auth.adobe.com/adobe-services/%s'
_USER_AGENT = 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0' _USER_AGENT = 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0'
_MODERN_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; rv:131.0) Gecko/20100101 Firefox/131.0'
_MVPD_CACHE = 'ap-mvpd' _MVPD_CACHE = 'ap-mvpd'
_DOWNLOADING_LOGIN_PAGE = 'Downloading Provider Login Page' _DOWNLOADING_LOGIN_PAGE = 'Downloading Provider Login Page'
@@ -1367,6 +1376,11 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
return super()._download_webpage_handle( return super()._download_webpage_handle(
*args, **kwargs) *args, **kwargs)
@staticmethod
def _get_mso_headers(mso_info):
# Not needed currently
return {}
@staticmethod @staticmethod
def _get_mvpd_resource(provider_id, title, guid, rating): def _get_mvpd_resource(provider_id, title, guid, rating):
channel = etree.Element('channel') channel = etree.Element('channel')
@@ -1382,7 +1396,13 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
resource_rating.text = rating resource_rating.text = rating
return '<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/">' + etree.tostring(channel).decode() + '</rss>' return '<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/">' + etree.tostring(channel).decode() + '</rss>'
def _extract_mvpd_auth(self, url, video_id, requestor_id, resource): def _extract_mvpd_auth(self, url, video_id, requestor_id, resource, software_statement):
mso_id = self.get_param('ap_mso')
if mso_id:
mso_info = MSO_INFO[mso_id]
else:
mso_info = {}
def xml_text(xml_str, tag): def xml_text(xml_str, tag):
return self._search_regex( return self._search_regex(
f'<{tag}>(.+?)</{tag}>', xml_str, tag) f'<{tag}>(.+?)</{tag}>', xml_str, tag)
@@ -1391,15 +1411,27 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
token_expires = unified_timestamp(re.sub(r'[_ ]GMT', '', xml_text(token, date_ele))) token_expires = unified_timestamp(re.sub(r'[_ ]GMT', '', xml_text(token, date_ele)))
return token_expires and token_expires <= int(time.time()) return token_expires and token_expires <= int(time.time())
def post_form(form_page_res, note, data={}): def post_form(form_page_res, note, data={}, validate_url=False):
form_page, urlh = form_page_res form_page, urlh = form_page_res
post_url = self._html_search_regex(r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_page, 'post url', group='url') post_url = self._html_search_regex(r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_page, 'post url', group='url')
if not re.match(r'https?://', post_url): if not re.match(r'https?://', post_url):
post_url = urllib.parse.urljoin(urlh.url, post_url) post_url = urllib.parse.urljoin(urlh.url, post_url)
if validate_url:
# This request is submitting credentials so we should validate it when possible
url_parsed = urllib.parse.urlparse(post_url)
expected_hostname = mso_info.get('login_hostname')
if expected_hostname and expected_hostname != url_parsed.hostname:
raise ExtractorError(
f'Unexpected login URL hostname; expected "{expected_hostname}" but got '
f'"{url_parsed.hostname}". Aborting before submitting credentials')
if url_parsed.scheme != 'https':
self.write_debug('Upgrading login URL scheme to https')
post_url = urllib.parse.urlunparse(url_parsed._replace(scheme='https'))
form_data = self._hidden_inputs(form_page) form_data = self._hidden_inputs(form_page)
form_data.update(data) form_data.update(data)
return self._download_webpage_handle( return self._download_webpage_handle(
post_url, video_id, note, data=urlencode_postdata(form_data), headers={ post_url, video_id, note, data=urlencode_postdata(form_data), headers={
**self._get_mso_headers(mso_info),
'Content-Type': 'application/x-www-form-urlencoded', 'Content-Type': 'application/x-www-form-urlencoded',
}) })
@@ -1432,40 +1464,72 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
} }
guid = xml_text(resource, 'guid') if '<' in resource else resource guid = xml_text(resource, 'guid') if '<' in resource else resource
count = 0 for _ in range(2):
while count < 2:
requestor_info = self.cache.load(self._MVPD_CACHE, requestor_id) or {} requestor_info = self.cache.load(self._MVPD_CACHE, requestor_id) or {}
authn_token = requestor_info.get('authn_token') authn_token = requestor_info.get('authn_token')
if authn_token and is_expired(authn_token, 'simpleTokenExpires'): if authn_token and is_expired(authn_token, 'simpleTokenExpires'):
authn_token = None authn_token = None
if not authn_token: if not authn_token:
mso_id = self.get_param('ap_mso') if not mso_id:
if mso_id: raise_mvpd_required()
username, password = self._get_login_info('ap_username', 'ap_password', mso_id) username, password = self._get_login_info('ap_username', 'ap_password', mso_id)
if not username or not password: if not username or not password:
raise_mvpd_required()
mso_info = MSO_INFO[mso_id]
provider_redirect_page_res = self._download_webpage_handle(
self._SERVICE_PROVIDER_TEMPLATE % 'authenticate/saml', video_id,
'Downloading Provider Redirect Page', query={
'noflash': 'true',
'mso_id': mso_id,
'requestor_id': requestor_id,
'no_iframe': 'false',
'domain_name': 'adobe.com',
'redirect_url': url,
}, headers={
# yt-dlp's default user-agent is usually too old for Comcast_SSO
# See: https://github.com/yt-dlp/yt-dlp/issues/10848
'User-Agent': self._MODERN_USER_AGENT,
} if mso_id == 'Comcast_SSO' else None)
elif not self._cookies_passed:
raise_mvpd_required() raise_mvpd_required()
if not mso_id: device_info, urlh = self._download_json_handle(
pass 'https://sp.auth.adobe.com/indiv/devices',
elif mso_id == 'Comcast_SSO': video_id, 'Registering device with Adobe',
data=json.dumps({'fingerprint': uuid.uuid4().hex}).encode(),
headers={'Content-Type': 'application/json; charset=UTF-8'})
device_id = device_info['deviceId']
mvpd_headers['pass_sfp'] = urlh.get_header('pass_sfp')
mvpd_headers['Ap_21'] = device_id
registration = self._download_json(
'https://sp.auth.adobe.com/o/client/register',
video_id, 'Registering client with Adobe',
data=json.dumps({'software_statement': software_statement}).encode(),
headers={'Content-Type': 'application/json; charset=UTF-8'})
access_token = self._download_json(
'https://sp.auth.adobe.com/o/client/token', video_id,
'Obtaining access token', data=urlencode_postdata({
'grant_type': 'client_credentials',
'client_id': registration['client_id'],
'client_secret': registration['client_secret'],
}),
headers={
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
})['access_token']
mvpd_headers['Authorization'] = f'Bearer {access_token}'
reg_code = self._download_json(
f'https://sp.auth.adobe.com/reggie/v1/{requestor_id}/regcode',
video_id, 'Obtaining registration code',
data=urlencode_postdata({
'requestor': requestor_id,
'deviceId': device_id,
'format': 'json',
}),
headers={
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Authorization': f'Bearer {access_token}',
})['code']
provider_redirect_page_res = self._download_webpage_handle(
self._SERVICE_PROVIDER_TEMPLATE % 'authenticate/saml', video_id,
'Downloading Provider Redirect Page', query={
'noflash': 'true',
'mso_id': mso_id,
'requestor_id': requestor_id,
'no_iframe': 'false',
'domain_name': 'adobe.com',
'redirect_url': url,
'reg_code': reg_code,
}, headers=self._get_mso_headers(mso_info))
if mso_id == 'Comcast_SSO':
# Comcast page flow varies by video site and whether you # Comcast page flow varies by video site and whether you
# are on Comcast's network. # are on Comcast's network.
provider_redirect_page, urlh = provider_redirect_page_res provider_redirect_page, urlh = provider_redirect_page_res
@@ -1489,8 +1553,8 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
oauth_redirect_url = extract_redirect_url( oauth_redirect_url = extract_redirect_url(
provider_redirect_page, fatal=True) provider_redirect_page, fatal=True)
provider_login_page_res = self._download_webpage_handle( provider_login_page_res = self._download_webpage_handle(
oauth_redirect_url, video_id, oauth_redirect_url, video_id, self._DOWNLOADING_LOGIN_PAGE,
self._DOWNLOADING_LOGIN_PAGE) headers=self._get_mso_headers(mso_info))
else: else:
provider_login_page_res = post_form( provider_login_page_res = post_form(
provider_redirect_page_res, provider_redirect_page_res,
@@ -1500,24 +1564,35 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
provider_login_page_res, 'Logging in', { provider_login_page_res, 'Logging in', {
mso_info['username_field']: username, mso_info['username_field']: username,
mso_info['password_field']: password, mso_info['password_field']: password,
}) }, validate_url=True)
mvpd_confirm_page, urlh = mvpd_confirm_page_res mvpd_confirm_page, urlh = mvpd_confirm_page_res
if '<button class="submit" value="Resume">Resume</button>' in mvpd_confirm_page: if '<button class="submit" value="Resume">Resume</button>' in mvpd_confirm_page:
post_form(mvpd_confirm_page_res, 'Confirming Login') post_form(mvpd_confirm_page_res, 'Confirming Login')
elif mso_id == 'Philo': elif mso_id == 'Philo':
# Philo has very unique authentication method # Philo has very unique authentication method
self._download_webpage( self._request_webpage(
'https://idp.philo.com/auth/init/login_code', video_id, 'Requesting auth code', data=urlencode_postdata({ 'https://idp.philo.com/auth/init/login_code', video_id,
'Requesting Philo auth code', data=json.dumps({
'ident': username, 'ident': username,
'device': 'web', 'device': 'web',
'send_confirm_link': False, 'send_confirm_link': False,
'send_token': True, 'send_token': True,
})) 'device_ident': f'web-{uuid.uuid4().hex}',
'include_login_link': True,
}).encode(), headers={
'Content-Type': 'application/json',
'Accept': 'application/json',
})
philo_code = getpass.getpass('Type auth code you have received [Return]: ') philo_code = getpass.getpass('Type auth code you have received [Return]: ')
self._download_webpage( self._request_webpage(
'https://idp.philo.com/auth/update/login_code', video_id, 'Submitting token', data=urlencode_postdata({ 'https://idp.philo.com/auth/update/login_code', video_id,
'token': philo_code, 'Submitting token', data=json.dumps({'token': philo_code}).encode(),
})) headers={
'Content-Type': 'application/json',
'Accept': 'application/json',
})
mvpd_confirm_page_res = self._download_webpage_handle('https://idp.philo.com/idp/submit', video_id, 'Confirming Philo Login') mvpd_confirm_page_res = self._download_webpage_handle('https://idp.philo.com/idp/submit', video_id, 'Confirming Philo Login')
post_form(mvpd_confirm_page_res, 'Confirming Login') post_form(mvpd_confirm_page_res, 'Confirming Login')
elif mso_id == 'Verizon': elif mso_id == 'Verizon':
@@ -1539,7 +1614,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
provider_redirect_page_res, 'Logging in', { provider_redirect_page_res, 'Logging in', {
mso_info['username_field']: username, mso_info['username_field']: username,
mso_info['password_field']: password, mso_info['password_field']: password,
}) }, validate_url=True)
saml_login_page, urlh = saml_login_page_res saml_login_page, urlh = saml_login_page_res
if 'Please try again.' in saml_login_page: if 'Please try again.' in saml_login_page:
raise ExtractorError( raise ExtractorError(
@@ -1560,7 +1635,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
[saml_login_page, saml_redirect_url], 'Logging in', { [saml_login_page, saml_redirect_url], 'Logging in', {
mso_info['username_field']: username, mso_info['username_field']: username,
mso_info['password_field']: password, mso_info['password_field']: password,
}) }, validate_url=True)
if 'Please try again.' in saml_login_page: if 'Please try again.' in saml_login_page:
raise ExtractorError( raise ExtractorError(
'Failed to login, incorrect User ID or Password.') 'Failed to login, incorrect User ID or Password.')
@@ -1631,7 +1706,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
provider_login_page_res, 'Logging in', { provider_login_page_res, 'Logging in', {
mso_info['username_field']: username, mso_info['username_field']: username,
mso_info['password_field']: password, mso_info['password_field']: password,
}) }, validate_url=True)
provider_refresh_redirect_url = extract_redirect_url( provider_refresh_redirect_url = extract_redirect_url(
provider_association_redirect, url=urlh.url) provider_association_redirect, url=urlh.url)
@@ -1682,7 +1757,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
provider_login_page_res, 'Logging in', { provider_login_page_res, 'Logging in', {
mso_info['username_field']: username, mso_info['username_field']: username,
mso_info['password_field']: password, mso_info['password_field']: password,
}) }, validate_url=True)
provider_refresh_redirect_url = extract_redirect_url( provider_refresh_redirect_url = extract_redirect_url(
provider_association_redirect, url=urlh.url) provider_association_redirect, url=urlh.url)
@@ -1699,6 +1774,27 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
query=hidden_data) query=hidden_data)
post_form(mvpd_confirm_page_res, 'Confirming Login') post_form(mvpd_confirm_page_res, 'Confirming Login')
elif mso_id == 'Fubo':
_, urlh = provider_redirect_page_res
fubo_response = self._download_json(
'https://api.fubo.tv/partners/tve/connect', video_id,
'Authenticating with Fubo', 'Unable to authenticate with Fubo',
query=parse_qs(urlh.url), data=json.dumps({
'username': username,
'password': password,
}).encode(), headers={
'Accept': 'application/json',
'Content-Type': 'application/json',
})
self._request_webpage(
'https://sp.auth.adobe.com/adobe-services/oauth2', video_id,
'Authenticating with Adobe', 'Failed to authenticate with Adobe',
query={
'code': fubo_response['code'],
'state': fubo_response['state'],
})
else: else:
# Some providers (e.g. DIRECTV NOW) have another meta refresh # Some providers (e.g. DIRECTV NOW) have another meta refresh
# based redirect that should be followed. # based redirect that should be followed.
@@ -1717,7 +1813,8 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
} }
if mso_id in ('Cablevision', 'AlticeOne'): if mso_id in ('Cablevision', 'AlticeOne'):
form_data['_eventId_proceed'] = '' form_data['_eventId_proceed'] = ''
mvpd_confirm_page_res = post_form(provider_login_page_res, 'Logging in', form_data) mvpd_confirm_page_res = post_form(
provider_login_page_res, 'Logging in', form_data, validate_url=True)
if mso_id != 'Rogers': if mso_id != 'Rogers':
post_form(mvpd_confirm_page_res, 'Confirming Login') post_form(mvpd_confirm_page_res, 'Confirming Login')
@@ -1727,6 +1824,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
'Retrieving Session', data=urlencode_postdata({ 'Retrieving Session', data=urlencode_postdata({
'_method': 'GET', '_method': 'GET',
'requestor_id': requestor_id, 'requestor_id': requestor_id,
'reg_code': reg_code,
}), headers=mvpd_headers) }), headers=mvpd_headers)
except ExtractorError as e: except ExtractorError as e:
if not mso_id and isinstance(e.cause, HTTPError) and e.cause.status == 401: if not mso_id and isinstance(e.cause, HTTPError) and e.cause.status == 401:
@@ -1734,7 +1832,6 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
raise raise
if '<pendingLogout' in session: if '<pendingLogout' in session:
self.cache.store(self._MVPD_CACHE, requestor_id, {}) self.cache.store(self._MVPD_CACHE, requestor_id, {})
count += 1
continue continue
authn_token = unescapeHTML(xml_text(session, 'authnToken')) authn_token = unescapeHTML(xml_text(session, 'authnToken'))
requestor_info['authn_token'] = authn_token requestor_info['authn_token'] = authn_token
@@ -1755,7 +1852,6 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
}), headers=mvpd_headers) }), headers=mvpd_headers)
if '<pendingLogout' in authorize: if '<pendingLogout' in authorize:
self.cache.store(self._MVPD_CACHE, requestor_id, {}) self.cache.store(self._MVPD_CACHE, requestor_id, {})
count += 1
continue continue
if '<error' in authorize: if '<error' in authorize:
raise ExtractorError(xml_text(authorize, 'details'), expected=True) raise ExtractorError(xml_text(authorize, 'details'), expected=True)
@@ -1778,6 +1874,5 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
}), headers=mvpd_headers) }), headers=mvpd_headers)
if '<pendingLogout' in short_authorize: if '<pendingLogout' in short_authorize:
self.cache.store(self._MVPD_CACHE, requestor_id, {}) self.cache.store(self._MVPD_CACHE, requestor_id, {})
count += 1
continue continue
return short_authorize return short_authorize
plugins/youtube_download/yt_dlp/extractor/adobetv.py
+70 -255
View File
@@ -1,285 +1,100 @@
import functools
import re
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import (
ISO639Utils, ISO639Utils,
OnDemandPagedList, clean_html,
determine_ext,
float_or_none, float_or_none,
int_or_none, int_or_none,
join_nonempty, join_nonempty,
parse_duration, url_or_none,
str_or_none,
str_to_int,
unified_strdate,
) )
from ..utils.traversal import traverse_obj
class AdobeTVBaseIE(InfoExtractor): class AdobeTVVideoIE(InfoExtractor):
def _call_api(self, path, video_id, query, note=None):
return self._download_json(
'http://tv.adobe.com/api/v4/' + path,
video_id, note, query=query)['data']
def _parse_subtitles(self, video_data, url_key):
subtitles = {}
for translation in video_data.get('translations', []):
vtt_path = translation.get(url_key)
if not vtt_path:
continue
lang = translation.get('language_w3c') or ISO639Utils.long2short(translation['language_medium'])
subtitles.setdefault(lang, []).append({
'ext': 'vtt',
'url': vtt_path,
})
return subtitles
def _parse_video_data(self, video_data):
video_id = str(video_data['id'])
title = video_data['title']
s3_extracted = False
formats = []
for source in video_data.get('videos', []):
source_url = source.get('url')
if not source_url:
continue
f = {
'format_id': source.get('quality_level'),
'fps': int_or_none(source.get('frame_rate')),
'height': int_or_none(source.get('height')),
'tbr': int_or_none(source.get('video_data_rate')),
'width': int_or_none(source.get('width')),
'url': source_url,
}
original_filename = source.get('original_filename')
if original_filename:
if not (f.get('height') and f.get('width')):
mobj = re.search(r'_(\d+)x(\d+)', original_filename)
if mobj:
f.update({
'height': int(mobj.group(2)),
'width': int(mobj.group(1)),
})
if original_filename.startswith('s3://') and not s3_extracted:
formats.append({
'format_id': 'original',
'quality': 1,
'url': original_filename.replace('s3://', 'https://s3.amazonaws.com/'),
})
s3_extracted = True
formats.append(f)
return {
'id': video_id,
'title': title,
'description': video_data.get('description'),
'thumbnail': video_data.get('thumbnail'),
'upload_date': unified_strdate(video_data.get('start_date')),
'duration': parse_duration(video_data.get('duration')),
'view_count': str_to_int(video_data.get('playcount')),
'formats': formats,
'subtitles': self._parse_subtitles(video_data, 'vtt'),
}
class AdobeTVEmbedIE(AdobeTVBaseIE):
IE_NAME = 'adobetv:embed'
_VALID_URL = r'https?://tv\.adobe\.com/embed/\d+/(?P<id>\d+)'
_TEST = {
'url': 'https://tv.adobe.com/embed/22/4153',
'md5': 'c8c0461bf04d54574fc2b4d07ac6783a',
'info_dict': {
'id': '4153',
'ext': 'flv',
'title': 'Creating Graphics Optimized for BlackBerry',
'description': 'md5:eac6e8dced38bdaae51cd94447927459',
'thumbnail': r're:https?://.*\.jpg$',
'upload_date': '20091109',
'duration': 377,
'view_count': int,
},
}
def _real_extract(self, url):
video_id = self._match_id(url)
video_data = self._call_api(
'episode/' + video_id, video_id, {'disclosure': 'standard'})[0]
return self._parse_video_data(video_data)
class AdobeTVIE(AdobeTVBaseIE):
IE_NAME = 'adobetv' IE_NAME = 'adobetv'
_VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?watch/(?P<show_urlname>[^/]+)/(?P<id>[^/]+)'
_TEST = {
'url': 'http://tv.adobe.com/watch/the-complete-picture-with-julieanne-kost/quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop/',
'md5': '9bc5727bcdd55251f35ad311ca74fa1e',
'info_dict': {
'id': '10981',
'ext': 'mp4',
'title': 'Quick Tip - How to Draw a Circle Around an Object in Photoshop',
'description': 'md5:99ec318dc909d7ba2a1f2b038f7d2311',
'thumbnail': r're:https?://.*\.jpg$',
'upload_date': '20110914',
'duration': 60,
'view_count': int,
},
}
def _real_extract(self, url):
language, show_urlname, urlname = self._match_valid_url(url).groups()
if not language:
language = 'en'
video_data = self._call_api(
'episode/get', urlname, {
'disclosure': 'standard',
'language': language,
'show_urlname': show_urlname,
'urlname': urlname,
})[0]
return self._parse_video_data(video_data)
class AdobeTVPlaylistBaseIE(AdobeTVBaseIE):
_PAGE_SIZE = 25
def _fetch_page(self, display_id, query, page):
page += 1
query['page'] = page
for element_data in self._call_api(
self._RESOURCE, display_id, query, f'Download Page {page}'):
yield self._process_data(element_data)
def _extract_playlist_entries(self, display_id, query):
return OnDemandPagedList(functools.partial(
self._fetch_page, display_id, query), self._PAGE_SIZE)
class AdobeTVShowIE(AdobeTVPlaylistBaseIE):
IE_NAME = 'adobetv:show'
_VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?show/(?P<id>[^/]+)'
_TEST = {
'url': 'http://tv.adobe.com/show/the-complete-picture-with-julieanne-kost',
'info_dict': {
'id': '36',
'title': 'The Complete Picture with Julieanne Kost',
'description': 'md5:fa50867102dcd1aa0ddf2ab039311b27',
},
'playlist_mincount': 136,
}
_RESOURCE = 'episode'
_process_data = AdobeTVBaseIE._parse_video_data
def _real_extract(self, url):
language, show_urlname = self._match_valid_url(url).groups()
if not language:
language = 'en'
query = {
'disclosure': 'standard',
'language': language,
'show_urlname': show_urlname,
}
show_data = self._call_api(
'show/get', show_urlname, query)[0]
return self.playlist_result(
self._extract_playlist_entries(show_urlname, query),
str_or_none(show_data.get('id')),
show_data.get('show_name'),
show_data.get('show_description'))
class AdobeTVChannelIE(AdobeTVPlaylistBaseIE):
IE_NAME = 'adobetv:channel'
_VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?channel/(?P<id>[^/]+)(?:/(?P<category_urlname>[^/]+))?'
_TEST = {
'url': 'http://tv.adobe.com/channel/development',
'info_dict': {
'id': 'development',
},
'playlist_mincount': 96,
}
_RESOURCE = 'show'
def _process_data(self, show_data):
return self.url_result(
show_data['url'], 'AdobeTVShow', str_or_none(show_data.get('id')))
def _real_extract(self, url):
language, channel_urlname, category_urlname = self._match_valid_url(url).groups()
if not language:
language = 'en'
query = {
'channel_urlname': channel_urlname,
'language': language,
}
if category_urlname:
query['category_urlname'] = category_urlname
return self.playlist_result(
self._extract_playlist_entries(channel_urlname, query),
channel_urlname)
class AdobeTVVideoIE(AdobeTVBaseIE):
IE_NAME = 'adobetv:video'
_VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P<id>\d+)' _VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P<id>\d+)'
_EMBED_REGEX = [r'<iframe[^>]+src=[\'"](?P<url>(?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]'] _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>(?:https?:)?//video\.tv\.adobe\.com/v/\d+)']
_TESTS = [{
_TEST = { 'url': 'https://video.tv.adobe.com/v/2456',
# From https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners
'url': 'https://video.tv.adobe.com/v/2456/',
'md5': '43662b577c018ad707a63766462b1e87', 'md5': '43662b577c018ad707a63766462b1e87',
'info_dict': { 'info_dict': {
'id': '2456', 'id': '2456',
'ext': 'mp4', 'ext': 'mp4',
'title': 'New experience with Acrobat DC', 'title': 'New experience with Acrobat DC',
'description': 'New experience with Acrobat DC', 'description': 'New experience with Acrobat DC',
'duration': 248.667, 'duration': 248.522,
'thumbnail': r're:https?://images-tv\.adobe\.com/.+\.jpg',
}, },
} }, {
'url': 'https://video.tv.adobe.com/v/3463980/adobe-acrobat',
'info_dict': {
'id': '3463980',
'ext': 'mp4',
'title': 'Adobe Acrobat: How to Customize the Toolbar for Faster PDF Editing',
'description': 'md5:94368ab95ae24f9c1bee0cb346e03dc3',
'duration': 97.514,
'thumbnail': r're:https?://images-tv\.adobe\.com/.+\.jpg',
},
}]
_WEBPAGE_TESTS = [{
# https://video.tv.adobe.com/v/3442499
'url': 'https://business.adobe.com/dx-fragments/summit/2025/marquees/S335/ondemand.live.html',
'info_dict': {
'id': '3442499',
'ext': 'mp4',
'title': 'S335 - Beyond Personalization: Creating Intent-Based Experiences at Scale',
'description': 'Beyond Personalization: Creating Intent-Based Experiences at Scale',
'duration': 2906.8,
'thumbnail': r're:https?://images-tv\.adobe\.com/.+\.jpg',
},
}]
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id) webpage = self._download_webpage(url, video_id)
video_data = self._search_json(
video_data = self._parse_json(self._search_regex( r'var\s+bridge\s*=', webpage, 'bridged data', video_id)
r'var\s+bridge\s*=\s*([^;]+);', webpage, 'bridged data'), video_id)
title = video_data['title']
formats = [] formats = []
sources = video_data.get('sources') or [] for source in traverse_obj(video_data, (
for source in sources: 'sources', lambda _, v: v['format'] != 'playlist' and url_or_none(v['src']),
source_src = source.get('src') )):
if not source_src: source_url = self._proto_relative_url(source['src'])
continue if determine_ext(source_url) == 'm3u8':
formats.append({ fmts = self._extract_m3u8_formats(
'filesize': int_or_none(source.get('kilobytes') or None, invscale=1000), source_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
'format_id': join_nonempty(source.get('format'), source.get('label')), else:
'height': int_or_none(source.get('height') or None), fmts = [{'url': source_url}]
'tbr': int_or_none(source.get('bitrate') or None),
'width': int_or_none(source.get('width') or None),
'url': source_src,
})
# For both metadata and downloaded files the duration varies among for fmt in fmts:
# formats. I just pick the max one fmt.update(traverse_obj(source, {
duration = max(filter(None, [ 'duration': ('duration', {float_or_none(scale=1000)}),
float_or_none(source.get('duration'), scale=1000) 'filesize': ('kilobytes', {float_or_none(invscale=1000)}),
for source in sources])) 'format_id': (('format', 'label'), {str}, all, {lambda x: join_nonempty(*x)}),
'height': ('height', {int_or_none}),
'tbr': ('bitrate', {int_or_none}),
'width': ('width', {int_or_none}),
}))
formats.extend(fmts)
subtitles = {}
for translation in traverse_obj(video_data, (
'translations', lambda _, v: url_or_none(v['vttPath']),
)):
lang = translation.get('language_w3c') or ISO639Utils.long2short(translation.get('language_medium')) or 'und'
subtitles.setdefault(lang, []).append({
'ext': 'vtt',
'url': self._proto_relative_url(translation['vttPath']),
})
return { return {
'id': video_id, 'id': video_id,
'formats': formats, 'formats': formats,
'title': title, 'subtitles': subtitles,
'description': video_data.get('description'), **traverse_obj(video_data, {
'thumbnail': video_data.get('video', {}).get('poster'), 'title': ('title', {clean_html}),
'duration': duration, 'description': ('description', {clean_html}, filter),
'subtitles': self._parse_subtitles(video_data, 'vttPath'), 'thumbnail': ('video', 'poster', {self._proto_relative_url}, {url_or_none}),
}),
} }
plugins/youtube_download/yt_dlp/extractor/adultswim.py
+3 -1
View File
@@ -84,6 +84,8 @@ class AdultSwimIE(TurnerBaseIE):
'skip': '404 Not Found', 'skip': '404 Not Found',
}] }]
_SOFTWARE_STATEMENT = 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIwNjg5ZmU2My00OTc5LTQxZmQtYWYxNC1hYjVlNmJjNWVkZWIiLCJuYmYiOjE1MzcxOTA2NzQsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTM3MTkwNjc0fQ.Xl3AEduM0s1TxDQ6-XssdKIiLm261hhsEv1C1yo_nitIajZThSI9rXILqtIzO0aujoHhdzUnu_dUCq9ffiSBzEG632tTa1la-5tegHtce80cMhewBN4n2t8n9O5tiaPx8MPY8ALdm5wS7QzWE6DO_LTJKgE8Bl7Yv-CWJT4q4SywtNiQWLVOuhBRnDyfsRezxRwptw8qTn9dv5ZzUrVJaby5fDZ_nOncMKvegOgaKd5KEuCAGQ-mg-PSuValMjGuf6FwDguGaK7IyI5Y2oOrzXmD4Dj7q4WBg8w9QoZhtLeAU56mcsGILolku2R5FHlVLO9xhjResyt-pfmegOkpSw'
def _real_extract(self, url): def _real_extract(self, url):
show_path, episode_path = self._match_valid_url(url).groups() show_path, episode_path = self._match_valid_url(url).groups()
display_id = episode_path or show_path display_id = episode_path or show_path
@@ -152,7 +154,7 @@ class AdultSwimIE(TurnerBaseIE):
# CDN_TOKEN_APP_ID from: # CDN_TOKEN_APP_ID from:
# https://d2gg02c3xr550i.cloudfront.net/assets/asvp.e9c8bef24322d060ef87.bundle.js # https://d2gg02c3xr550i.cloudfront.net/assets/asvp.e9c8bef24322d060ef87.bundle.js
'appId': 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhcHBJZCI6ImFzLXR2ZS1kZXNrdG9wLXB0enQ2bSIsInByb2R1Y3QiOiJ0dmUiLCJuZXR3b3JrIjoiYXMiLCJwbGF0Zm9ybSI6ImRlc2t0b3AiLCJpYXQiOjE1MzI3MDIyNzl9.BzSCk-WYOZ2GMCIaeVb8zWnzhlgnXuJTCu0jGp_VaZE', 'appId': 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhcHBJZCI6ImFzLXR2ZS1kZXNrdG9wLXB0enQ2bSIsInByb2R1Y3QiOiJ0dmUiLCJuZXR3b3JrIjoiYXMiLCJwbGF0Zm9ybSI6ImRlc2t0b3AiLCJpYXQiOjE1MzI3MDIyNzl9.BzSCk-WYOZ2GMCIaeVb8zWnzhlgnXuJTCu0jGp_VaZE',
}, { }, self._SOFTWARE_STATEMENT, {
'url': url, 'url': url,
'site_name': 'AdultSwim', 'site_name': 'AdultSwim',
'auth_required': auth, 'auth_required': auth,
plugins/youtube_download/yt_dlp/extractor/aenetworks.py
+76 -33
View File
@@ -1,3 +1,5 @@
import json
from .theplatform import ThePlatformIE from .theplatform import ThePlatformIE
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
@@ -6,7 +8,6 @@ from ..utils import (
remove_start, remove_start,
traverse_obj, traverse_obj,
update_url_query, update_url_query,
urlencode_postdata,
) )
@@ -20,13 +21,13 @@ class AENetworksBaseIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
_THEPLATFORM_KEY = '43jXaGRQud' _THEPLATFORM_KEY = '43jXaGRQud'
_THEPLATFORM_SECRET = 'S10BPXHMlb' _THEPLATFORM_SECRET = 'S10BPXHMlb'
_DOMAIN_MAP = { _DOMAIN_MAP = {
'history.com': ('HISTORY', 'history'), 'history.com': ('HISTORY', 'history', 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiI1MzZlMTQ3ZS0zMzFhLTQxY2YtYTMwNC01MDA2NzNlOGYwYjYiLCJuYmYiOjE1Mzg2NjMzMDksImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTM4NjYzMzA5fQ.n24-FVHLGXJe2D4atIQZ700aiXKIajKh5PWFoHJ40Az4itjtwwSFHnvufnoal3T8lYkwNLxce7H-IEGxIykRkZEdwq09pMKMT-ft9ASzE4vQ8fAWbf5ZgDME86x4Jq_YaxkRc9Ne0eShGhl8fgTJHvk07sfWcol61HJ7kU7K8FzzcHR0ucFQgA5VNd8RyjoGWY7c6VxnXR214LOpXsywmit04-vGJC102b_WA2EQfqI93UzG6M6l0EeV4n0_ijP3s8_i8WMJZ_uwnTafCIY6G_731i01dKXDLSFzG1vYglAwDa8DTcdrAAuIFFDF6QNGItCCmwbhjufjmoeVb7R1Gg'),
'aetv.com': ('AETV', 'aetv'), 'aetv.com': ('AETV', 'aetv', 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiI5Y2IwNjg2Yy03ODUxLTRiZDUtODcyMC00MjNlZTg1YTQ1NzMiLCJuYmYiOjE1Mzg2NjMyOTAsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTM4NjYzMjkwfQ.T5Elf0X4TndO4NEgqBas1gDxNHGPVk_daO2Ha5FBzVO6xi3zM7eavdAKfYMCN7gpWYJx03iADaVPtczO_t_aGZczDjpwJHgTUzDgvcLZAVsVDqtDIAMy3S846rPgT6UDbVoxurA7B2VTPm9phjrSXhejvd0LBO8MQL4AZ3sy2VmiPJ2noT1ily5PuHCYlkrT1fheO064duR__Cd9DQ5VTMnKjzY3Cx345CEwKDkUk5gwgxhXM-aY0eblehrq8VD81_aRM_O3tvh7nbTydHOnUpV-k_iKVi49gqz7Sf8zb6Zh5z2Uftn3vYCfE5NQuesitoRMnsH17nW7o_D59hkRgg'),
'mylifetime.com': ('LIFETIME', 'lifetime'), 'mylifetime.com': ('LIFETIME', 'lifetime', 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJmODg0MDM1ZC1mZGRmLTRmYjgtYmRkMC05MzRhZDdiYTAwYTciLCJuYmYiOjE1NDkzOTI2NDQsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTQ5MzkyNjQ0fQ.vkTIaCpheKdKQd__2-3ec4qkcpbAhyCTvwe5iTl922ItSQfVhpEJG4wseVSNmBTrpBi0hvLedcw6Hj1_UuzBMVuVcCqLprU-pI8recEwL0u7G-eVkylsxe1OTUm1o3V6OykXQ9KlA-QQLL1neUhdhR1n5B1LZ4cmtBmiEpfgf4rFwXD1ScFylIcaWKLBqHoRBNUmxyTmoXXvn_A-GGSj9eCizFzY8W5uBwUcsoiw2Cr1skx7PbB2RSP1I5DsoIJKG-8XV1KS7MWl-fNLjE-hVAsI9znqfEEFcPBiv3LhCP4Nf4OIs7xAselMn0M0c8igRUZhURWX_hdygUAxkbKFtQ'),
'lifetimemovieclub.com': ('LIFETIMEMOVIECLUB', 'lmc'), 'fyi.tv': ('FYI', 'fyi', 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIxOGZiOWM3Ny1mYmMzLTQxYTktYmE1Yi1lMzM0ZmUzNzU4NjEiLCJuYmYiOjE1ODc1ODAzNzcsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTg3NTgwMzc3fQ.AYDuipKswmIfLBfOjHRsfc5fMV5NmJUmiJnkpiep4VEw9QiXkygFj4bN06Si5tFc5Mee5TDrGzDpV6iuKbVpLT5kuqXhAn-Wozf5zKPsg_IpdEKO7gsiCq4calt72ct44KTqtKD_hVcoxQU24_HaJsRgXzu3B-6Ff6UrmsXkyvYifYVC9v2DSkdCuA02_IrlllzVT2kRuefUXgL4vQRtTFf77uYa0RKSTG7uVkiQ_AU41eXevKlO2qgtc14Hk5cZ7-ZNrDyMCXYA5ngdIHP7Gs9PWaFXT36PFHI_rC4EfxUABPzjQFxjpP75aX5qn8SH__HbM9q3hoPWgaEaf76qIQ'),
'fyi.tv': ('FYI', 'fyi'), 'lifetimemovieclub.com': ('LIFETIMEMOVIECLUB', 'lmc', None),
'historyvault.com': (None, 'historyvault'), 'historyvault.com': (None, 'historyvault', None),
'biography.com': (None, 'biography'), 'biography.com': (None, 'biography', None),
} }
def _extract_aen_smil(self, smil_url, video_id, auth=None): def _extract_aen_smil(self, smil_url, video_id, auth=None):
@@ -71,7 +72,7 @@ class AENetworksBaseIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
} }
def _extract_aetn_info(self, domain, filter_key, filter_value, url): def _extract_aetn_info(self, domain, filter_key, filter_value, url):
requestor_id, brand = self._DOMAIN_MAP[domain] requestor_id, brand, software_statement = self._DOMAIN_MAP[domain]
result = self._download_json( result = self._download_json(
f'https://feeds.video.aetnd.com/api/v2/{brand}/videos', f'https://feeds.video.aetnd.com/api/v2/{brand}/videos',
filter_value, query={f'filter[{filter_key}]': filter_value}) filter_value, query={f'filter[{filter_key}]': filter_value})
@@ -95,7 +96,7 @@ class AENetworksBaseIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
theplatform_metadata.get('AETN$PPL_pplProgramId') or theplatform_metadata.get('AETN$PPL_pplProgramId_OLD'), theplatform_metadata.get('AETN$PPL_pplProgramId') or theplatform_metadata.get('AETN$PPL_pplProgramId_OLD'),
traverse_obj(theplatform_metadata, ('ratings', 0, 'rating'))) traverse_obj(theplatform_metadata, ('ratings', 0, 'rating')))
auth = self._extract_mvpd_auth( auth = self._extract_mvpd_auth(
url, video_id, requestor_id, resource) url, video_id, requestor_id, resource, software_statement)
info.update(self._extract_aen_smil(media_url, video_id, auth)) info.update(self._extract_aen_smil(media_url, video_id, auth))
info.update({ info.update({
'title': title, 'title': title,
@@ -110,11 +111,9 @@ class AENetworksIE(AENetworksBaseIE):
IE_NAME = 'aenetworks' IE_NAME = 'aenetworks'
IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network and History Vault' IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network and History Vault'
_VALID_URL = AENetworksBaseIE._BASE_URL_REGEX + r'''(?P<id> _VALID_URL = AENetworksBaseIE._BASE_URL_REGEX + r'''(?P<id>
shows/[^/]+/season-\d+/episode-\d+| shows/[^/?#]+/season-\d+/episode-\d+|
(?: (?P<type>movie|special)s/[^/?#]+(?P<extra>/[^/?#]+)?|
(?:movie|special)s/[^/]+| (?:shows/[^/?#]+/)?videos/[^/?#]+
(?:shows/[^/]+/)?videos
)/[^/?#&]+
)''' )'''
_TESTS = [{ _TESTS = [{
'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1', 'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
@@ -127,20 +126,18 @@ class AENetworksIE(AENetworksBaseIE):
'upload_date': '20120529', 'upload_date': '20120529',
'uploader': 'AENE-NEW', 'uploader': 'AENE-NEW',
'duration': 2592.0, 'duration': 2592.0,
'thumbnail': r're:^https?://.*\.jpe?g$', 'thumbnail': r're:https?://.+/.+\.jpg',
'chapters': 'count:5', 'chapters': 'count:5',
'tags': 'count:14', 'tags': 'count:14',
'categories': ['Mountain Men'], 'categories': ['Mountain Men'],
'episode_number': 1, 'episode_number': 1,
'episode': 'Episode 1', 'episode': 'Winter Is Coming',
'season': 'Season 1', 'season': 'Season 1',
'season_number': 1, 'season_number': 1,
'series': 'Mountain Men', 'series': 'Mountain Men',
'age_limit': 0,
}, },
'params': { 'params': {'skip_download': 'm3u8'},
# m3u8 download
'skip_download': True,
},
'add_ie': ['ThePlatform'], 'add_ie': ['ThePlatform'],
'skip': 'Geo-restricted - This content is not available in your location.', 'skip': 'Geo-restricted - This content is not available in your location.',
}, { }, {
@@ -154,21 +151,64 @@ class AENetworksIE(AENetworksBaseIE):
'upload_date': '20160112', 'upload_date': '20160112',
'uploader': 'AENE-NEW', 'uploader': 'AENE-NEW',
'duration': 1277.695, 'duration': 1277.695,
'thumbnail': r're:^https?://.*\.jpe?g$', 'thumbnail': r're:https?://.+/.+\.jpg',
'chapters': 'count:4', 'chapters': 'count:4',
'tags': 'count:23', 'tags': 'count:23',
'episode': 'Episode 1', 'episode': 'Inlawful Entry',
'episode_number': 1, 'episode_number': 1,
'season': 'Season 9', 'season': 'Season 9',
'season_number': 9, 'season_number': 9,
'series': 'Duck Dynasty', 'series': 'Duck Dynasty',
'age_limit': 0,
}, },
'params': { 'params': {'skip_download': 'm3u8'},
# m3u8 download 'add_ie': ['ThePlatform'],
'skip_download': True, }, {
}, 'url': 'https://play.mylifetime.com/movies/v-c-andrews-web-of-dreams',
'info_dict': {
'id': '1590627395981',
'ext': 'mp4',
'title': 'VC Andrews\' Web of Dreams',
'description': 'md5:2a8ba13ae64271c79eb65c0577d312ce',
'uploader': 'AENE-NEW',
'age_limit': 14,
'duration': 5253.665,
'thumbnail': r're:https?://.+/.+\.jpg',
'chapters': 'count:8',
'tags': ['lifetime', 'mylifetime', 'lifetime channel', "VC Andrews' Web of Dreams"],
'series': '',
'season': 'Season 0',
'season_number': 0,
'episode': 'VC Andrews\' Web of Dreams',
'episode_number': 0,
'timestamp': 1566489703.0,
'upload_date': '20190822',
},
'params': {'skip_download': 'm3u8'},
'add_ie': ['ThePlatform'],
}, {
'url': 'https://www.aetv.com/specials/hunting-jonbenets-killer-the-untold-story',
'info_dict': {
'id': '1488235587551',
'ext': 'mp4',
'title': 'Hunting JonBenet\'s Killer: The Untold Story',
'description': 'md5:209869425ee392d74fe29201821e48b4',
'uploader': 'AENE-NEW',
'age_limit': 14,
'duration': 5003.903,
'thumbnail': r're:https?://.+/.+\.jpg',
'chapters': 'count:10',
'tags': 'count:11',
'series': '',
'season': 'Season 0',
'season_number': 0,
'episode': 'Hunting JonBenet\'s Killer: The Untold Story',
'episode_number': 0,
'timestamp': 1554987697.0,
'upload_date': '20190411',
},
'params': {'skip_download': 'm3u8'},
'add_ie': ['ThePlatform'], 'add_ie': ['ThePlatform'],
'skip': 'This video is only available for users of participating TV providers.',
}, { }, {
'url': 'http://www.fyi.tv/shows/tiny-house-nation/season-1/episode-8', 'url': 'http://www.fyi.tv/shows/tiny-house-nation/season-1/episode-8',
'only_matching': True, 'only_matching': True,
@@ -196,25 +236,28 @@ class AENetworksIE(AENetworksBaseIE):
}] }]
def _real_extract(self, url): def _real_extract(self, url):
domain, canonical = self._match_valid_url(url).groups() domain, canonical, url_type, extra = self._match_valid_url(url).group('domain', 'id', 'type', 'extra')
if url_type in ('movie', 'special') and not extra:
canonical += f'/full-{url_type}'
return self._extract_aetn_info(domain, 'canonical', '/' + canonical, url) return self._extract_aetn_info(domain, 'canonical', '/' + canonical, url)
class AENetworksListBaseIE(AENetworksBaseIE): class AENetworksListBaseIE(AENetworksBaseIE):
def _call_api(self, resource, slug, brand, fields): def _call_api(self, resource, slug, brand, fields):
return self._download_json( return self._download_json(
'https://yoga.appsvcs.aetnd.com/graphql', 'https://yoga.appsvcs.aetnd.com/graphql', slug,
slug, query={'brand': brand}, data=urlencode_postdata({ query={'brand': brand}, headers={'Content-Type': 'application/json'},
data=json.dumps({
'query': '''{ 'query': '''{
%s(slug: "%s") { %s(slug: "%s") {
%s %s
} }
}''' % (resource, slug, fields), # noqa: UP031 }''' % (resource, slug, fields), # noqa: UP031
}))['data'][resource] }).encode())['data'][resource]
def _real_extract(self, url): def _real_extract(self, url):
domain, slug = self._match_valid_url(url).groups() domain, slug = self._match_valid_url(url).groups()
_, brand = self._DOMAIN_MAP[domain] _, brand, _ = self._DOMAIN_MAP[domain]
playlist = self._call_api(self._RESOURCE, slug, brand, self._FIELDS) playlist = self._call_api(self._RESOURCE, slug, brand, self._FIELDS)
base_url = f'http://watch.{domain}' base_url = f'http://watch.{domain}'
plugins/youtube_download/yt_dlp/extractor/amcnetworks.py
+27 -113
View File
@@ -1,32 +1,24 @@
import re from .brightcove import BrightcoveNewIE
from .common import InfoExtractor
from .theplatform import ThePlatformIE from ..utils.traversal import traverse_obj
from ..utils import (
int_or_none,
parse_age_limit,
try_get,
update_url_query,
)
class AMCNetworksIE(ThePlatformIE): # XXX: Do not subclass from concrete IE class AMCNetworksIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?(?P<site>amc|bbcamerica|ifc|(?:we|sundance)tv)\.com/(?P<id>(?:movies|shows(?:/[^/]+)+)/[^/?#&]+)' _VALID_URL = r'https?://(?:www\.)?(?:amc|bbcamerica|ifc|(?:we|sundance)tv)\.com/(?P<id>(?:movies|shows(?:/[^/?#]+)+)/[^/?#&]+)'
_TESTS = [{ _TESTS = [{
'url': 'https://www.bbcamerica.com/shows/the-graham-norton-show/videos/tina-feys-adorable-airline-themed-family-dinner--51631', 'url': 'https://www.amc.com/shows/dark-winds/videos/dark-winds-a-look-at-season-3--1072027',
'info_dict': { 'info_dict': {
'id': '4Lq1dzOnZGt0', 'id': '6369261343112',
'ext': 'mp4', 'ext': 'mp4',
'title': "The Graham Norton Show - Season 28 - Tina Fey's Adorable Airline-Themed Family Dinner", 'title': 'Dark Winds: A Look at Season 3',
'description': "It turns out child stewardesses are very generous with the wine! All-new episodes of 'The Graham Norton Show' premiere Fridays at 11/10c on BBC America.", 'uploader_id': '6240731308001',
'upload_date': '20201120', 'duration': 176.427,
'timestamp': 1605904350, 'thumbnail': r're:https://[^/]+\.boltdns\.net/.+/image\.jpg',
'uploader': 'AMCN', 'tags': [],
'timestamp': 1740414792,
'upload_date': '20250224',
}, },
'params': { 'params': {'skip_download': 'm3u8'},
# m3u8 download
'skip_download': True,
},
'skip': '404 Not Found',
}, { }, {
'url': 'http://www.bbcamerica.com/shows/the-hunt/full-episodes/season-1/episode-01-the-hardest-challenge', 'url': 'http://www.bbcamerica.com/shows/the-hunt/full-episodes/season-1/episode-01-the-hardest-challenge',
'only_matching': True, 'only_matching': True,
@@ -52,96 +44,18 @@ class AMCNetworksIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
'url': 'https://www.sundancetv.com/shows/riviera/full-episodes/season-1/episode-01-episode-1', 'url': 'https://www.sundancetv.com/shows/riviera/full-episodes/season-1/episode-01-episode-1',
'only_matching': True, 'only_matching': True,
}] }]
_REQUESTOR_ID_MAP = {
'amc': 'AMC',
'bbcamerica': 'BBCA',
'ifc': 'IFC',
'sundancetv': 'SUNDANCE',
'wetv': 'WETV',
}
def _real_extract(self, url): def _real_extract(self, url):
site, display_id = self._match_valid_url(url).groups() display_id = self._match_id(url)
requestor_id = self._REQUESTOR_ID_MAP[site] webpage = self._download_webpage(url, display_id)
page_data = self._download_json( initial_data = self._search_json(
f'https://content-delivery-gw.svc.ds.amcn.com/api/v2/content/amcn/{requestor_id.lower()}/url/{display_id}', r'window\.initialData\s*=\s*JSON\.parse\(String\.raw`', webpage, 'initial data', display_id)
display_id)['data'] video_id = traverse_obj(initial_data, ('initialData', 'properties', 'videoId', {str}))
properties = page_data.get('properties') or {} if not video_id: # All locked videos are now DRM-protected
query = { self.report_drm(display_id)
'mbr': 'true', account_id = initial_data['config']['brightcove']['accountId']
'manifest': 'm3u', player_id = initial_data['config']['brightcove']['playerId']
}
video_player_count = 0 return self.url_result(
try: f'https://players.brightcove.net/{account_id}/{player_id}_default/index.html?videoId={video_id}',
for v in page_data['children']: BrightcoveNewIE, video_id)
if v.get('type') == 'video-player':
release_pid = v['properties']['currentVideo']['meta']['releasePid']
tp_path = 'M_UwQC/' + release_pid
media_url = 'https://link.theplatform.com/s/' + tp_path
video_player_count += 1
except KeyError:
pass
if video_player_count > 1:
self.report_warning(
f'The JSON data has {video_player_count} video players. Only one will be extracted')
# Fall back to videoPid if releasePid not found.
# TODO: Fall back to videoPid if releasePid manifest uses DRM.
if not video_player_count:
tp_path = 'M_UwQC/media/' + properties['videoPid']
media_url = 'https://link.theplatform.com/s/' + tp_path
theplatform_metadata = self._download_theplatform_metadata(tp_path, display_id)
info = self._parse_theplatform_metadata(theplatform_metadata)
video_id = theplatform_metadata['pid']
title = theplatform_metadata['title']
rating = try_get(
theplatform_metadata, lambda x: x['ratings'][0]['rating'])
video_category = properties.get('videoCategory')
if video_category and video_category.endswith('-Auth'):
resource = self._get_mvpd_resource(
requestor_id, title, video_id, rating)
query['auth'] = self._extract_mvpd_auth(
url, video_id, requestor_id, resource)
media_url = update_url_query(media_url, query)
formats, subtitles = self._extract_theplatform_smil(
media_url, video_id)
thumbnails = []
thumbnail_urls = [properties.get('imageDesktop')]
if 'thumbnail' in info:
thumbnail_urls.append(info.pop('thumbnail'))
for thumbnail_url in thumbnail_urls:
if not thumbnail_url:
continue
mobj = re.search(r'(\d+)x(\d+)', thumbnail_url)
thumbnails.append({
'url': thumbnail_url,
'width': int(mobj.group(1)) if mobj else None,
'height': int(mobj.group(2)) if mobj else None,
})
info.update({
'age_limit': parse_age_limit(rating),
'formats': formats,
'id': video_id,
'subtitles': subtitles,
'thumbnails': thumbnails,
})
ns_keys = theplatform_metadata.get('$xmlns', {}).keys()
if ns_keys:
ns = next(iter(ns_keys))
episode = theplatform_metadata.get(ns + '$episodeTitle') or None
episode_number = int_or_none(
theplatform_metadata.get(ns + '$episode'))
season_number = int_or_none(
theplatform_metadata.get(ns + '$season'))
series = theplatform_metadata.get(ns + '$show') or None
info.update({
'episode': episode,
'episode_number': episode_number,
'season_number': season_number,
'series': series,
})
return info
plugins/youtube_download/yt_dlp/extractor/apa.py
+10 -2
View File
@@ -11,12 +11,11 @@ class APAIE(InfoExtractor):
_EMBED_REGEX = [r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//[^/]+\.apa\.at/embed/[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}.*?)\1'] _EMBED_REGEX = [r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//[^/]+\.apa\.at/embed/[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}.*?)\1']
_TESTS = [{ _TESTS = [{
'url': 'http://uvp.apa.at/embed/293f6d17-692a-44e3-9fd5-7b178f3a1029', 'url': 'http://uvp.apa.at/embed/293f6d17-692a-44e3-9fd5-7b178f3a1029',
'md5': '2b12292faeb0a7d930c778c7a5b4759b',
'info_dict': { 'info_dict': {
'id': '293f6d17-692a-44e3-9fd5-7b178f3a1029', 'id': '293f6d17-692a-44e3-9fd5-7b178f3a1029',
'ext': 'mp4', 'ext': 'mp4',
'title': '293f6d17-692a-44e3-9fd5-7b178f3a1029', 'title': '293f6d17-692a-44e3-9fd5-7b178f3a1029',
'thumbnail': r're:^https?://.*\.jpg$', 'thumbnail': r're:https?://kf-vn\.sf\.apa\.at/vn/.+\.jpg',
}, },
}, { }, {
'url': 'https://uvp-apapublisher.sf.apa.at/embed/2f94e9e6-d945-4db2-9548-f9a41ebf7b78', 'url': 'https://uvp-apapublisher.sf.apa.at/embed/2f94e9e6-d945-4db2-9548-f9a41ebf7b78',
@@ -28,6 +27,15 @@ class APAIE(InfoExtractor):
'url': 'http://uvp-kleinezeitung.sf.apa.at/embed/f1c44979-dba2-4ebf-b021-e4cf2cac3c81', 'url': 'http://uvp-kleinezeitung.sf.apa.at/embed/f1c44979-dba2-4ebf-b021-e4cf2cac3c81',
'only_matching': True, 'only_matching': True,
}] }]
_WEBPAGE_TESTS = [{
'url': 'https://www.vol.at/blue-man-group/5593454',
'info_dict': {
'id': '293f6d17-692a-44e3-9fd5-7b178f3a1029',
'ext': 'mp4',
'title': '293f6d17-692a-44e3-9fd5-7b178f3a1029',
'thumbnail': r're:https?://kf-vn\.sf\.apa\.at/vn/.+\.jpg',
},
}]
def _real_extract(self, url): def _real_extract(self, url):
mobj = self._match_valid_url(url) mobj = self._match_valid_url(url)
plugins/youtube_download/yt_dlp/extractor/appleconnect.py
+103 -25
View File
@@ -1,47 +1,125 @@
import time
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ExtractorError, str_to_int from ..utils import (
ExtractorError,
extract_attributes,
float_or_none,
jwt_decode_hs256,
jwt_encode,
parse_resolution,
qualities,
unified_strdate,
update_url,
url_or_none,
urljoin,
)
from ..utils.traversal import (
find_element,
require,
traverse_obj,
)
class AppleConnectIE(InfoExtractor): class AppleConnectIE(InfoExtractor):
_VALID_URL = r'https?://itunes\.apple\.com/\w{0,2}/?post/(?:id)?sa\.(?P<id>[\w-]+)' IE_NAME = 'apple:music:connect'
IE_DESC = 'Apple Music Connect'
_BASE_URL = 'https://music.apple.com'
_QUALITIES = {
'provisionalUploadVideo': None,
'sdVideo': 480,
'sdVideoWithPlusAudio': 480,
'sd480pVideo': 480,
'720pHdVideo': 720,
'1080pHdVideo': 1080,
}
_VALID_URL = r'https?://music\.apple\.com/[\w-]+/post/(?P<id>\d+)'
_TESTS = [{ _TESTS = [{
'url': 'https://itunes.apple.com/us/post/idsa.4ab17a39-2720-11e5-96c5-a5b38f6c42d3', 'url': 'https://music.apple.com/us/post/1018290019',
'md5': 'c1d41f72c8bcaf222e089434619316e4',
'info_dict': { 'info_dict': {
'id': '4ab17a39-2720-11e5-96c5-a5b38f6c42d3', 'id': '1018290019',
'ext': 'm4v', 'ext': 'm4v',
'title': 'Energy', 'title': 'Energy',
'uploader': 'Drake', 'duration': 177.911,
'thumbnail': r're:^https?://.*\.jpg$', 'thumbnail': r're:https?://.+\.png',
'upload_date': '20150710', 'upload_date': '20150710',
'timestamp': 1436545535, 'uploader': 'Drake',
}, },
}, { }, {
'url': 'https://itunes.apple.com/us/post/sa.0fe0229f-2457-11e5-9f40-1bb645f2d5d9', 'url': 'https://music.apple.com/us/post/1016746627',
'only_matching': True, 'info_dict': {
'id': '1016746627',
'ext': 'm4v',
'title': 'Body Shop (Madonna) - Chellous Lima (Acoustic Cover)',
'duration': 210.278,
'thumbnail': r're:https?://.+\.png',
'upload_date': '20150706',
'uploader': 'Chellous Lima',
},
}] }]
_jwt = None
@staticmethod
def _jwt_is_expired(token):
return jwt_decode_hs256(token)['exp'] - time.time() < 120
def _get_token(self, webpage, video_id):
if self._jwt and not self._jwt_is_expired(self._jwt):
return self._jwt
js_url = traverse_obj(webpage, (
{find_element(tag='script', attr='crossorigin', value='', html=True)},
{extract_attributes}, 'src', {urljoin(self._BASE_URL)}, {require('JS URL')}))
js = self._download_webpage(
js_url, video_id, 'Downloading token JS', 'Unable to download token JS')
header = jwt_encode({}, '', headers={'alg': 'ES256', 'kid': 'WebPlayKid'}).split('.')[0]
self._jwt = self._search_regex(
fr'(["\'])(?P<jwt>{header}(?:\.[\w-]+){{2}})\1', js, 'JSON Web Token', group='jwt')
if self._jwt_is_expired(self._jwt):
raise ExtractorError('The fetched token is already expired')
return self._jwt
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id) webpage = self._download_webpage(url, video_id)
try: videos = self._download_json(
video_json = self._html_search_regex( 'https://amp-api.music.apple.com/v1/catalog/us/uploaded-videos',
r'class="auc-video-data">(\{.*?\})', webpage, 'json') video_id, headers={
except ExtractorError: 'Authorization': f'Bearer {self._get_token(webpage, video_id)}',
raise ExtractorError('This post doesn\'t contain a video', expected=True) 'Origin': self._BASE_URL,
}, query={'ids': video_id, 'l': 'en-US'})
attributes = traverse_obj(videos, (
'data', ..., 'attributes', any, {require('video information')}))
video_data = self._parse_json(video_json, video_id) formats = []
timestamp = str_to_int(self._html_search_regex(r'data-timestamp="(\d+)"', webpage, 'timestamp')) quality = qualities(list(self._QUALITIES.keys()))
like_count = str_to_int(self._html_search_regex(r'(\d+) Loves', webpage, 'like count', default=None)) for format_id, src_url in traverse_obj(attributes, (
'assetTokens', {dict.items}, lambda _, v: url_or_none(v[1]),
)):
formats.append({
'ext': 'm4v',
'format_id': format_id,
'height': self._QUALITIES.get(format_id),
'quality': quality(format_id),
'url': src_url,
**parse_resolution(update_url(src_url, query=None), lenient=True),
})
return { return {
'id': video_id, 'id': video_id,
'url': video_data['sslSrc'], 'formats': formats,
'title': video_data['title'], 'thumbnail': self._html_search_meta(
'description': video_data['description'], ['og:image', 'og:image:secure_url', 'twitter:image'], webpage),
'uploader': video_data['artistName'], **traverse_obj(attributes, {
'thumbnail': video_data['artworkUrl'], 'title': ('name', {str}),
'timestamp': timestamp, 'duration': ('durationInMilliseconds', {float_or_none(scale=1000)}),
'like_count': like_count, 'upload_date': ('uploadDate', {unified_strdate}),
'uploader': (('artistName', 'uploadingArtistName'), {str}, any),
'webpage_url': ('postUrl', {url_or_none}),
}),
} }
plugins/youtube_download/yt_dlp/extractor/applepodcasts.py
+5 -3
View File
@@ -1,5 +1,6 @@
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import (
clean_html,
clean_podcast_url, clean_podcast_url,
int_or_none, int_or_none,
parse_iso8601, parse_iso8601,
@@ -17,7 +18,7 @@ class ApplePodcastsIE(InfoExtractor):
'ext': 'mp3', 'ext': 'mp3',
'title': 'Ferreck Dawn - To The Break of Dawn 117', 'title': 'Ferreck Dawn - To The Break of Dawn 117',
'episode': 'Ferreck Dawn - To The Break of Dawn 117', 'episode': 'Ferreck Dawn - To The Break of Dawn 117',
'description': 'md5:1fc571102f79dbd0a77bfd71ffda23bc', 'description': 'md5:8c4f5c2c30af17ed6a98b0b9daf15b76',
'upload_date': '20240812', 'upload_date': '20240812',
'timestamp': 1723449600, 'timestamp': 1723449600,
'duration': 3596, 'duration': 3596,
@@ -58,7 +59,7 @@ class ApplePodcastsIE(InfoExtractor):
r'<script [^>]*\bid=["\']serialized-server-data["\'][^>]*>', webpage, r'<script [^>]*\bid=["\']serialized-server-data["\'][^>]*>', webpage,
'server data', episode_id, contains_pattern=r'\[{(?s:.+)}\]')[0]['data'] 'server data', episode_id, contains_pattern=r'\[{(?s:.+)}\]')[0]['data']
model_data = traverse_obj(server_data, ( model_data = traverse_obj(server_data, (
'headerButtonItems', lambda _, v: v['$kind'] == 'bookmark' and v['modelType'] == 'EpisodeOffer', 'headerButtonItems', lambda _, v: v['$kind'] == 'share' and v['modelType'] == 'EpisodeLockup',
'model', {dict}, any)) 'model', {dict}, any))
return { return {
@@ -68,7 +69,8 @@ class ApplePodcastsIE(InfoExtractor):
or self._yield_json_ld(webpage, episode_id, fatal=False), episode_id, fatal=False), or self._yield_json_ld(webpage, episode_id, fatal=False), episode_id, fatal=False),
**traverse_obj(model_data, { **traverse_obj(model_data, {
'title': ('title', {str}), 'title': ('title', {str}),
'url': ('streamUrl', {clean_podcast_url}), 'description': ('summary', {clean_html}),
'url': ('playAction', 'episodeOffer', 'streamUrl', {clean_podcast_url}),
'timestamp': ('releaseDate', {parse_iso8601}), 'timestamp': ('releaseDate', {parse_iso8601}),
'duration': ('duration', {int_or_none}), 'duration': ('duration', {int_or_none}),
}), }),
plugins/youtube_download/yt_dlp/extractor/archiveorg.py
+53 -31
View File
@@ -16,6 +16,7 @@ from ..utils import (
dict_get, dict_get,
extract_attributes, extract_attributes,
get_element_by_id, get_element_by_id,
get_element_text_and_html_by_tag,
int_or_none, int_or_none,
join_nonempty, join_nonempty,
js_to_json, js_to_json,
@@ -32,7 +33,6 @@ from ..utils import (
unified_timestamp, unified_timestamp,
url_or_none, url_or_none,
urlhandle_detect_ext, urlhandle_detect_ext,
variadic,
) )
@@ -72,6 +72,7 @@ class ArchiveOrgIE(InfoExtractor):
'display_id': 'Cops-v2.mp4', 'display_id': 'Cops-v2.mp4',
'thumbnail': r're:https://archive\.org/download/.*\.jpg', 'thumbnail': r're:https://archive\.org/download/.*\.jpg',
'duration': 1091.96, 'duration': 1091.96,
'track': 'Cops-v2',
}, },
}, { }, {
'url': 'http://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellect', 'url': 'http://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
@@ -86,6 +87,7 @@ class ArchiveOrgIE(InfoExtractor):
'thumbnail': r're:https://archive\.org/download/.*\.jpg', 'thumbnail': r're:https://archive\.org/download/.*\.jpg',
'duration': 59.77, 'duration': 59.77,
'display_id': 'Commercial-JFK1960ElectionAdCampaignJingle.mpg', 'display_id': 'Commercial-JFK1960ElectionAdCampaignJingle.mpg',
'track': 'Commercial-JFK1960ElectionAdCampaignJingle',
}, },
}, { }, {
'url': 'https://archive.org/details/Election_Ads/Commercial-Nixon1960ElectionAdToughonDefense.mpg', 'url': 'https://archive.org/details/Election_Ads/Commercial-Nixon1960ElectionAdToughonDefense.mpg',
@@ -102,6 +104,7 @@ class ArchiveOrgIE(InfoExtractor):
'duration': 59.51, 'duration': 59.51,
'license': 'http://creativecommons.org/licenses/publicdomain/', 'license': 'http://creativecommons.org/licenses/publicdomain/',
'thumbnail': r're:https://archive\.org/download/.*\.jpg', 'thumbnail': r're:https://archive\.org/download/.*\.jpg',
'track': 'Commercial-Nixon1960ElectionAdToughonDefense',
}, },
}, { }, {
'url': 'https://archive.org/details/gd1977-05-08.shure57.stevenson.29303.flac16', 'url': 'https://archive.org/details/gd1977-05-08.shure57.stevenson.29303.flac16',
@@ -182,6 +185,7 @@ class ArchiveOrgIE(InfoExtractor):
'duration': 130.46, 'duration': 130.46,
'thumbnail': 'https://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel1_01_000117.jpg', 'thumbnail': 'https://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel1_01_000117.jpg',
'display_id': 'irelandthemakingofarepublicreel1_01.mov', 'display_id': 'irelandthemakingofarepublicreel1_01.mov',
'track': 'irelandthemakingofarepublicreel1 01',
}, },
}, { }, {
'md5': '67335ee3b23a0da930841981c1e79b02', 'md5': '67335ee3b23a0da930841981c1e79b02',
@@ -192,6 +196,7 @@ class ArchiveOrgIE(InfoExtractor):
'title': 'irelandthemakingofarepublicreel1_02.mov', 'title': 'irelandthemakingofarepublicreel1_02.mov',
'display_id': 'irelandthemakingofarepublicreel1_02.mov', 'display_id': 'irelandthemakingofarepublicreel1_02.mov',
'thumbnail': 'https://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel1_02_001374.jpg', 'thumbnail': 'https://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel1_02_001374.jpg',
'track': 'irelandthemakingofarepublicreel1 02',
}, },
}, { }, {
'md5': 'e470e86787893603f4a341a16c281eb5', 'md5': 'e470e86787893603f4a341a16c281eb5',
@@ -202,6 +207,7 @@ class ArchiveOrgIE(InfoExtractor):
'title': 'irelandthemakingofarepublicreel2.mov', 'title': 'irelandthemakingofarepublicreel2.mov',
'thumbnail': 'https://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel2_001554.jpg', 'thumbnail': 'https://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel2_001554.jpg',
'display_id': 'irelandthemakingofarepublicreel2.mov', 'display_id': 'irelandthemakingofarepublicreel2.mov',
'track': 'irelandthemakingofarepublicreel2',
}, },
}, },
], ],
@@ -225,19 +231,29 @@ class ArchiveOrgIE(InfoExtractor):
'release_date': '19950402', 'release_date': '19950402',
'timestamp': 1084927901, 'timestamp': 1084927901,
}, },
}, {
# metadata['metadata']['description'] is a list of strings instead of str
'url': 'https://archive.org/details/pra-KZ1908.02',
'info_dict': {
'id': 'pra-KZ1908.02',
'ext': 'mp3',
'display_id': 'KZ1908.02_01.wav',
'title': 'Crips and Bloods speak about gang life',
'description': 'md5:2b56b35ff021311e3554b47a285e70b3',
'uploader': '[email protected]',
'duration': 1733.74,
'track': 'KZ1908.02 01',
'track_number': 1,
'timestamp': 1336026026,
'upload_date': '20120503',
'release_year': 1992,
},
}] }]
@staticmethod @staticmethod
def _playlist_data(webpage): def _playlist_data(webpage):
element = re.findall(r'''(?xs) element = get_element_text_and_html_by_tag('play-av', webpage)[1]
<input return json.loads(extract_attributes(element)['playlist'])
(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
\s+class=['"]?js-play8-playlist['"]?
(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
\s*/>
''', webpage)[0]
return json.loads(extract_attributes(element)['value'])
def _real_extract(self, url): def _real_extract(self, url):
video_id = urllib.parse.unquote_plus(self._match_id(url)) video_id = urllib.parse.unquote_plus(self._match_id(url))
@@ -274,34 +290,40 @@ class ArchiveOrgIE(InfoExtractor):
m = metadata['metadata'] m = metadata['metadata']
identifier = m['identifier'] identifier = m['identifier']
info = { info = traverse_obj(m, {
'title': ('title', {str}),
'description': ('description', ({str}, (..., all, {' '.join})), {clean_html}, filter, any),
'uploader': (('uploader', 'adder'), {str}, any),
'creators': ('creator', (None, ...), {str}, filter, all, filter),
'license': ('licenseurl', {url_or_none}),
'release_date': ('date', {unified_strdate}),
'timestamp': (('publicdate', 'addeddate'), {unified_timestamp}, any),
'location': ('venue', {str}),
'release_year': ('year', {int_or_none}),
})
info.update({
'id': identifier, 'id': identifier,
'title': m['title'],
'description': clean_html(m.get('description')),
'uploader': dict_get(m, ['uploader', 'adder']),
'creators': traverse_obj(m, ('creator', {variadic}, {lambda x: x[0] and list(x)})),
'license': m.get('licenseurl'),
'release_date': unified_strdate(m.get('date')),
'timestamp': unified_timestamp(dict_get(m, ['publicdate', 'addeddate'])),
'webpage_url': f'https://archive.org/details/{identifier}', 'webpage_url': f'https://archive.org/details/{identifier}',
'location': m.get('venue'), })
'release_year': int_or_none(m.get('year'))}
for f in metadata['files']: for f in metadata['files']:
if f['name'] in entries: if f['name'] in entries:
entries[f['name']] = merge_dicts(entries[f['name']], { entries[f['name']] = merge_dicts(entries[f['name']], {
'id': identifier + '/' + f['name'], 'id': identifier + '/' + f['name'],
'title': f.get('title') or f['name'], **traverse_obj(f, {
'display_id': f['name'], 'title': (('title', 'name'), {str}, any),
'description': clean_html(f.get('description')), 'display_id': ('name', {str}),
'creators': traverse_obj(f, ('creator', {variadic}, {lambda x: x[0] and list(x)})), 'description': ('description', ({str}, (..., all, {' '.join})), {clean_html}, filter, any),
'duration': parse_duration(f.get('length')), 'creators': ('creator', (None, ...), {str}, filter, all, filter),
'track_number': int_or_none(f.get('track')), 'duration': ('length', {parse_duration}),
'album': f.get('album'), 'track_number': ('track', {int_or_none}),
'discnumber': int_or_none(f.get('disc')), 'album': ('album', {str}),
'release_year': int_or_none(f.get('year'))}) 'discnumber': ('disc', {int_or_none}),
'release_year': ('year', {int_or_none}),
}),
})
entry = entries[f['name']] entry = entries[f['name']]
elif traverse_obj(f, 'original', expected_type=str) in entries: elif traverse_obj(f, ('original', {str})) in entries:
entry = entries[f['original']] entry = entries[f['original']]
else: else:
continue continue
@@ -718,7 +740,7 @@ class YoutubeWebArchiveIE(InfoExtractor):
note or 'Downloading CDX API JSON', query=query, fatal=fatal) note or 'Downloading CDX API JSON', query=query, fatal=fatal)
if isinstance(res, list) and len(res) >= 2: if isinstance(res, list) and len(res) >= 2:
# format response to make it easier to use # format response to make it easier to use
return [dict(zip(res[0], v)) for v in res[1:]] return [dict(zip(res[0], v)) for v in res[1:]] # noqa: B905
elif not isinstance(res, list) or len(res) != 0: elif not isinstance(res, list) or len(res) != 0:
self.report_warning('Error while parsing CDX API response' + bug_reports_message()) self.report_warning('Error while parsing CDX API response' + bug_reports_message())
plugins/youtube_download/yt_dlp/extractor/arcpublishing.py
+14
View File
@@ -62,6 +62,20 @@ class ArcPublishingIE(InfoExtractor):
'url': 'arcpublishing:tronc:460f2931-8130-4719-8ea1-ffcb2d7cb685', 'url': 'arcpublishing:tronc:460f2931-8130-4719-8ea1-ffcb2d7cb685',
'only_matching': True, 'only_matching': True,
}] }]
_WEBPAGE_TESTS = [{
'url': 'https://www.uppermichiganssource.com/2025/07/18/scattered-showers-storms-bring-heavy-rain-potential/',
'info_dict': {
'id': '508116f7-e999-48db-b7c2-60a04842679b',
'ext': 'mp4',
'title': 'Scattered showers & storms bring heavy rain potential',
'description': 'Scattered showers & storms bring heavy rain potential',
'duration': 2016,
'thumbnail': r're:https?://.+\.jpg',
'timestamp': 1752881287,
'upload_date': '20250718',
},
'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'],
}]
_POWA_DEFAULTS = [ _POWA_DEFAULTS = [
(['cmg', 'prisa'], '%s-config-prod.api.cdn.arcpublishing.com/video'), (['cmg', 'prisa'], '%s-config-prod.api.cdn.arcpublishing.com/video'),
([ ([
plugins/youtube_download/yt_dlp/extractor/arkena.py
-150
View File
@@ -1,150 +0,0 @@
from .common import InfoExtractor
from ..utils import (
ExtractorError,
float_or_none,
int_or_none,
parse_iso8601,
parse_qs,
try_get,
)
class ArkenaIE(InfoExtractor):
_VALID_URL = r'''(?x)
https?://
(?:
video\.(?:arkena|qbrick)\.com/play2/embed/player\?|
play\.arkena\.com/(?:config|embed)/avp/v\d/player/media/(?P<id>[^/]+)/[^/]+/(?P<account_id>\d+)
)
'''
# See https://support.arkena.com/display/PLAY/Ways+to+embed+your+video
_EMBED_REGEX = [r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//play\.arkena\.com/embed/avp/.+?)\1']
_TESTS = [{
'url': 'https://video.qbrick.com/play2/embed/player?accountId=1034090&mediaId=d8ab4607-00090107-aab86310',
'md5': '97f117754e5f3c020f5f26da4a44ebaf',
'info_dict': {
'id': 'd8ab4607-00090107-aab86310',
'ext': 'mp4',
'title': 'EM_HT20_117_roslund_v2.mp4',
'timestamp': 1608285912,
'upload_date': '20201218',
'duration': 1429.162667,
'subtitles': {
'sv': 'count:3',
},
},
}, {
'url': 'https://play.arkena.com/embed/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411',
'only_matching': True,
}, {
'url': 'https://play.arkena.com/config/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411/?callbackMethod=jQuery1111023664739129262213_1469227693893',
'only_matching': True,
}, {
'url': 'http://play.arkena.com/config/avp/v1/player/media/327336/darkmatter/131064/?callbackMethod=jQuery1111002221189684892677_1469227595972',
'only_matching': True,
}, {
'url': 'http://play.arkena.com/embed/avp/v1/player/media/327336/darkmatter/131064/',
'only_matching': True,
}, {
'url': 'http://video.arkena.com/play2/embed/player?accountId=472718&mediaId=35763b3b-00090078-bf604299&pageStyling=styled',
'only_matching': True,
}]
def _real_extract(self, url):
mobj = self._match_valid_url(url)
video_id = mobj.group('id')
account_id = mobj.group('account_id')
# Handle http://video.arkena.com/play2/embed/player URL
if not video_id:
qs = parse_qs(url)
video_id = qs.get('mediaId', [None])[0]
account_id = qs.get('accountId', [None])[0]
if not video_id or not account_id:
raise ExtractorError('Invalid URL', expected=True)
media = self._download_json(
f'https://video.qbrick.com/api/v1/public/accounts/{account_id}/medias/{video_id}',
video_id, query={
# https://video.qbrick.com/docs/api/examples/library-api.html
'fields': 'asset/resources/*/renditions/*(height,id,language,links/*(href,mimeType),type,size,videos/*(audios/*(codec,sampleRate),bitrate,codec,duration,height,width),width),created,metadata/*(title,description),tags',
})
metadata = media.get('metadata') or {}
title = metadata['title']
duration = None
formats = []
thumbnails = []
subtitles = {}
for resource in media['asset']['resources']:
for rendition in (resource.get('renditions') or []):
rendition_type = rendition.get('type')
for i, link in enumerate(rendition.get('links') or []):
href = link.get('href')
if not href:
continue
if rendition_type == 'image':
thumbnails.append({
'filesize': int_or_none(rendition.get('size')),
'height': int_or_none(rendition.get('height')),
'id': rendition.get('id'),
'url': href,
'width': int_or_none(rendition.get('width')),
})
elif rendition_type == 'subtitle':
subtitles.setdefault(rendition.get('language') or 'en', []).append({
'url': href,
})
elif rendition_type == 'video':
f = {
'filesize': int_or_none(rendition.get('size')),
'format_id': rendition.get('id'),
'url': href,
}
video = try_get(rendition, lambda x: x['videos'][i], dict)
if video:
if not duration:
duration = float_or_none(video.get('duration'))
f.update({
'height': int_or_none(video.get('height')),
'tbr': int_or_none(video.get('bitrate'), 1000),
'vcodec': video.get('codec'),
'width': int_or_none(video.get('width')),
})
audio = try_get(video, lambda x: x['audios'][0], dict)
if audio:
f.update({
'acodec': audio.get('codec'),
'asr': int_or_none(audio.get('sampleRate')),
})
formats.append(f)
elif rendition_type == 'index':
mime_type = link.get('mimeType')
if mime_type == 'application/smil+xml':
formats.extend(self._extract_smil_formats(
href, video_id, fatal=False))
elif mime_type == 'application/x-mpegURL':
formats.extend(self._extract_m3u8_formats(
href, video_id, 'mp4', 'm3u8_native',
m3u8_id='hls', fatal=False))
elif mime_type == 'application/hds+xml':
formats.extend(self._extract_f4m_formats(
href, video_id, f4m_id='hds', fatal=False))
elif mime_type == 'application/dash+xml':
formats.extend(self._extract_mpd_formats(
href, video_id, mpd_id='dash', fatal=False))
elif mime_type == 'application/vnd.ms-sstr+xml':
formats.extend(self._extract_ism_formats(
href, video_id, ism_id='mss', fatal=False))
return {
'id': video_id,
'title': title,
'description': metadata.get('description'),
'timestamp': parse_iso8601(media.get('created')),
'thumbnails': thumbnails,
'subtitles': subtitles,
'duration': duration,
'tags': media.get('tags'),
'formats': formats,
}
plugins/youtube_download/yt_dlp/extractor/arte.py
+29 -4
View File
@@ -51,8 +51,8 @@ class ArteTVIE(ArteTVBaseIE):
'id': '109067-000-A', 'id': '109067-000-A',
'ext': 'mp4', 'ext': 'mp4',
'description': 'md5:d2ca367b8ecee028dddaa8bd1aebc739', 'description': 'md5:d2ca367b8ecee028dddaa8bd1aebc739',
'thumbnail': r're:https?://api-cdn\.arte\.tv/img/v2/image/.+',
'timestamp': 1713927600, 'timestamp': 1713927600,
'thumbnail': 'https://api-cdn.arte.tv/img/v2/image/3rR6PLzfbigSkkeHtkCZNF/940x530',
'duration': 7599, 'duration': 7599,
'title': 'La loi de Téhéran', 'title': 'La loi de Téhéran',
'upload_date': '20240424', 'upload_date': '20240424',
@@ -62,6 +62,7 @@ class ArteTVIE(ArteTVBaseIE):
'fr-forced': 'mincount:1', 'fr-forced': 'mincount:1',
}, },
}, },
'skip': 'Invalid URL',
}, { }, {
'note': 'age-restricted', 'note': 'age-restricted',
'url': 'https://www.arte.tv/de/videos/006785-000-A/the-element-of-crime/', 'url': 'https://www.arte.tv/de/videos/006785-000-A/the-element-of-crime/',
@@ -69,9 +70,9 @@ class ArteTVIE(ArteTVBaseIE):
'id': '006785-000-A', 'id': '006785-000-A',
'description': 'md5:c2f94fdfefc8a280e4dab68ab96ab0ba', 'description': 'md5:c2f94fdfefc8a280e4dab68ab96ab0ba',
'title': 'The Element of Crime', 'title': 'The Element of Crime',
'thumbnail': r're:https?://api-cdn\.arte\.tv/img/v2/image/.+',
'timestamp': 1696111200, 'timestamp': 1696111200,
'duration': 5849, 'duration': 5849,
'thumbnail': 'https://api-cdn.arte.tv/img/v2/image/q82dTTfyuCXupPsGxXsd7B/940x530',
'upload_date': '20230930', 'upload_date': '20230930',
'ext': 'mp4', 'ext': 'mp4',
}, },
@@ -252,6 +253,30 @@ class ArteTVEmbedIE(InfoExtractor):
'url': 'https://www.arte.tv/player/v3/index.php?json_url=https://api.arte.tv/api/player/v2/config/de/100605-013-A', 'url': 'https://www.arte.tv/player/v3/index.php?json_url=https://api.arte.tv/api/player/v2/config/de/100605-013-A',
'only_matching': True, 'only_matching': True,
}] }]
_WEBPAGE_TESTS = [{
# FIXME: Embed detection
'url': 'https://timesofmalta.com/article/watch-sunken-warships-north-sea-arte.1108358',
'info_dict': {
'id': '110288-000-A',
'ext': 'mp4',
'title': 'Danger on the Seabed',
'alt_title': 'Sunken Warships in the North Sea',
'description': 'md5:a2c84cbad37d280bddb6484087120add',
'duration': 3148,
'thumbnail': r're:https?://api-cdn\.arte\.tv/img/v2/image/.+',
'timestamp': 1741686820,
'upload_date': '20250311',
},
'params': {'skip_download': 'm3u8'},
}, {
# FIXME: Embed detection
'url': 'https://www.eurockeennes.fr/en-live/',
'info_dict': {
'id': 'en-live',
'title': 'Les Eurocks en live | Les Eurockéennes de Belfort – 3-4-5-6 juillet 2025 sur la Presqu&#039;Île du Malsaucy',
},
'playlist_count': 4,
}]
def _real_extract(self, url): def _real_extract(self, url):
qs = parse_qs(url) qs = parse_qs(url)
@@ -304,9 +329,9 @@ class ArteTVCategoryIE(ArteTVBaseIE):
'info_dict': { 'info_dict': {
'id': 'politics-and-society', 'id': 'politics-and-society',
'title': 'Politics and society', 'title': 'Politics and society',
'description': 'Investigative documentary series, geopolitical analysis, and international commentary', 'description': 'Watch documentaries and reportage about politics, society and current affairs.',
}, },
'playlist_mincount': 13, 'playlist_mincount': 3,
}] }]
@classmethod @classmethod
plugins/youtube_download/yt_dlp/extractor/atvat.py
+2 -3
View File
@@ -4,7 +4,7 @@ from .common import InfoExtractor
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
float_or_none, float_or_none,
jwt_encode_hs256, jwt_encode,
try_get, try_get,
) )
@@ -83,11 +83,10 @@ class ATVAtIE(InfoExtractor):
'nbf': int(not_before.timestamp()), 'nbf': int(not_before.timestamp()),
'exp': int(expire.timestamp()), 'exp': int(expire.timestamp()),
} }
jwt_token = jwt_encode_hs256(payload, self._ENCRYPTION_KEY, headers={'kid': self._ACCESS_ID})
videos = self._download_json( videos = self._download_json(
'https://vas-v4.p7s1video.net/4.0/getsources', 'https://vas-v4.p7s1video.net/4.0/getsources',
content_id, 'Downloading videos JSON', query={ content_id, 'Downloading videos JSON', query={
'token': jwt_token.decode('utf-8'), 'token': jwt_encode(payload, self._ENCRYPTION_KEY, headers={'kid': self._ACCESS_ID}),
}) })
video_id, videos_data = next(iter(videos['data'].items())) video_id, videos_data = next(iter(videos['data'].items()))
plugins/youtube_download/yt_dlp/extractor/bandaichannel.py
-33
View File
@@ -1,33 +0,0 @@
from .brightcove import BrightcoveNewBaseIE
from ..utils import extract_attributes
class BandaiChannelIE(BrightcoveNewBaseIE):
IE_NAME = 'bandaichannel'
_VALID_URL = r'https?://(?:www\.)?b-ch\.com/titles/(?P<id>\d+/\d+)'
_TESTS = [{
'url': 'https://www.b-ch.com/titles/514/001',
'md5': 'a0f2d787baa5729bed71108257f613a4',
'info_dict': {
'id': '6128044564001',
'ext': 'mp4',
'title': 'メタルファイターMIKU 第1話',
'timestamp': 1580354056,
'uploader_id': '5797077852001',
'upload_date': '20200130',
'duration': 1387.733,
},
'params': {
'skip_download': True,
},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
attrs = extract_attributes(self._search_regex(
r'(<video-js[^>]+\bid="bcplayer"[^>]*>)', webpage, 'player'))
bc = self._download_json(
'https://pbifcd.b-ch.com/v1/playbackinfo/ST/70/' + attrs['data-info'],
video_id, headers={'X-API-KEY': attrs['data-auth'].strip()})['bc']
return self._parse_brightcove_metadata(bc, bc['id'])
plugins/youtube_download/yt_dlp/extractor/bandcamp.py
+42 -16
View File
@@ -7,6 +7,7 @@ from .common import InfoExtractor
from ..utils import ( from ..utils import (
KNOWN_EXTENSIONS, KNOWN_EXTENSIONS,
ExtractorError, ExtractorError,
clean_html,
extract_attributes, extract_attributes,
float_or_none, float_or_none,
int_or_none, int_or_none,
@@ -19,7 +20,7 @@ from ..utils import (
url_or_none, url_or_none,
urljoin, urljoin,
) )
from ..utils.traversal import find_element, traverse_obj from ..utils.traversal import find_element, find_elements, traverse_obj
class BandcampIE(InfoExtractor): class BandcampIE(InfoExtractor):
@@ -35,14 +36,12 @@ class BandcampIE(InfoExtractor):
'duration': 9.8485, 'duration': 9.8485,
'uploader': 'youtube-dl "\'/\\ä↭', 'uploader': 'youtube-dl "\'/\\ä↭',
'upload_date': '20121129', 'upload_date': '20121129',
'thumbnail': r're:https?://f4\.bcbits\.com/img/.+\.jpg',
'timestamp': 1354224127, 'timestamp': 1354224127,
'track': 'youtube-dl "\'/\\ä↭ - youtube-dl test song "\'/\\ä↭', 'track': 'youtube-dl "\'/\\ä↭ - youtube-dl test song "\'/\\ä↭',
'album_artist': 'youtube-dl "\'/\\ä↭',
'track_id': '1812978515', 'track_id': '1812978515',
'artist': 'youtube-dl "\'/\\ä↭',
'uploader_url': 'https://youtube-dl.bandcamp.com', 'uploader_url': 'https://youtube-dl.bandcamp.com',
'uploader_id': 'youtube-dl', 'uploader_id': 'youtube-dl',
'thumbnail': 'https://f4.bcbits.com/img/a3216802731_5.jpg',
'artists': ['youtube-dl "\'/\\ä↭'], 'artists': ['youtube-dl "\'/\\ä↭'],
'album_artists': ['youtube-dl "\'/\\ä↭'], 'album_artists': ['youtube-dl "\'/\\ä↭'],
}, },
@@ -53,10 +52,9 @@ class BandcampIE(InfoExtractor):
'info_dict': { 'info_dict': {
'id': '2650410135', 'id': '2650410135',
'ext': 'm4a', 'ext': 'm4a',
'acodec': r're:[fa]lac',
'title': 'Ben Prunty - Lanius (Battle)', 'title': 'Ben Prunty - Lanius (Battle)',
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'Ben Prunty', 'uploader': 'Ben Prunty',
'thumbnail': r're:https?://f4\.bcbits\.com/img/.+\.jpg',
'timestamp': 1396508491, 'timestamp': 1396508491,
'upload_date': '20140403', 'upload_date': '20140403',
'release_timestamp': 1396483200, 'release_timestamp': 1396483200,
@@ -65,11 +63,12 @@ class BandcampIE(InfoExtractor):
'track': 'Lanius (Battle)', 'track': 'Lanius (Battle)',
'track_number': 1, 'track_number': 1,
'track_id': '2650410135', 'track_id': '2650410135',
'artist': 'Ben Prunty',
'album_artist': 'Ben Prunty',
'album': 'FTL: Advanced Edition Soundtrack', 'album': 'FTL: Advanced Edition Soundtrack',
'uploader_url': 'https://benprunty.bandcamp.com', 'uploader_url': 'https://benprunty.bandcamp.com',
'uploader_id': 'benprunty', 'uploader_id': 'benprunty',
'tags': ['soundtrack', 'chiptunes', 'cinematic', 'electronic', 'video game music', 'California'],
'artists': ['Ben Prunty'],
'album_artists': ['Ben Prunty'],
}, },
}, { }, {
# no free download, mp3 128 # no free download, mp3 128
@@ -79,8 +78,8 @@ class BandcampIE(InfoExtractor):
'id': '2584466013', 'id': '2584466013',
'ext': 'mp3', 'ext': 'mp3',
'title': 'Mastodon - Hail to Fire', 'title': 'Mastodon - Hail to Fire',
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'Mastodon', 'uploader': 'Mastodon',
'thumbnail': r're:https?://f4\.bcbits\.com/img/.+\.jpg',
'timestamp': 1322005399, 'timestamp': 1322005399,
'upload_date': '20111122', 'upload_date': '20111122',
'release_timestamp': 1076112000, 'release_timestamp': 1076112000,
@@ -89,11 +88,12 @@ class BandcampIE(InfoExtractor):
'track': 'Hail to Fire', 'track': 'Hail to Fire',
'track_number': 5, 'track_number': 5,
'track_id': '2584466013', 'track_id': '2584466013',
'artist': 'Mastodon',
'album_artist': 'Mastodon',
'album': 'Call of the Mastodon', 'album': 'Call of the Mastodon',
'uploader_url': 'https://relapsealumni.bandcamp.com', 'uploader_url': 'https://relapsealumni.bandcamp.com',
'uploader_id': 'relapsealumni', 'uploader_id': 'relapsealumni',
'tags': ['Philadelphia'],
'artists': ['Mastodon'],
'album_artists': ['Mastodon'],
}, },
}, { }, {
# track from compilation album (artist/album_artist difference) # track from compilation album (artist/album_artist difference)
@@ -103,8 +103,8 @@ class BandcampIE(InfoExtractor):
'id': '1978174799', 'id': '1978174799',
'ext': 'mp3', 'ext': 'mp3',
'title': 'submerse - submerse - Safehouse', 'title': 'submerse - submerse - Safehouse',
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'submerse', 'uploader': 'submerse',
'thumbnail': r're:https?://f4\.bcbits\.com/img/.+\.jpg',
'timestamp': 1480779297, 'timestamp': 1480779297,
'upload_date': '20161203', 'upload_date': '20161203',
'release_timestamp': 1481068800, 'release_timestamp': 1481068800,
@@ -113,11 +113,36 @@ class BandcampIE(InfoExtractor):
'track': 'submerse - Safehouse', 'track': 'submerse - Safehouse',
'track_number': 3, 'track_number': 3,
'track_id': '1978174799', 'track_id': '1978174799',
'artist': 'submerse',
'album_artist': 'Diskotopia',
'album': 'DSK F/W 2016-2017 Free Compilation', 'album': 'DSK F/W 2016-2017 Free Compilation',
'uploader_url': 'https://diskotopia.bandcamp.com', 'uploader_url': 'https://diskotopia.bandcamp.com',
'uploader_id': 'diskotopia', 'uploader_id': 'diskotopia',
'tags': ['Japan'],
'artists': ['submerse'],
'album_artists': ['Diskotopia'],
},
}]
_WEBPAGE_TESTS = [{
# FIXME: Embed detection
'url': 'https://www.punknews.org/article/85809/stay-inside-super-sonic',
'info_dict': {
'id': '2475540375',
'ext': 'mp3',
'title': 'Stay Inside - Super Sonic',
'album': 'Lunger',
'album_artists': ['Stay Inside'],
'artists': ['Stay Inside'],
'duration': 166.157,
'release_date': '20251003',
'release_timestamp': 1759449600.0,
'thumbnail': r're:https?://f4\.bcbits\.com/img/.+\.jpg',
'timestamp': 1749473029.0,
'track': 'Super Sonic',
'track_id': '2475540375',
'track_number': 3,
'upload_date': '20250609',
'uploader': 'Stay Inside',
'uploader_id': 'stayinside',
'uploader_url': 'https://stayinside.bandcamp.com',
}, },
}] }]
@@ -252,6 +277,7 @@ class BandcampIE(InfoExtractor):
'album': embed.get('album_title'), 'album': embed.get('album_title'),
'album_artist': album_artist, 'album_artist': album_artist,
'formats': formats, 'formats': formats,
'tags': traverse_obj(webpage, ({find_elements(cls='tag')}, ..., {clean_html})),
} }
@@ -268,10 +294,10 @@ class BandcampAlbumIE(BandcampIE): # XXX: Do not subclass from concrete IE
'id': '1353101989', 'id': '1353101989',
'ext': 'mp3', 'ext': 'mp3',
'title': 'Blazo - Intro', 'title': 'Blazo - Intro',
'thumbnail': r're:https?://f4\.bcbits\.com/img/.+\.jpg',
'timestamp': 1311756226, 'timestamp': 1311756226,
'upload_date': '20110727', 'upload_date': '20110727',
'uploader': 'Blazo', 'uploader': 'Blazo',
'thumbnail': 'https://f4.bcbits.com/img/a1721150828_5.jpg',
'album_artists': ['Blazo'], 'album_artists': ['Blazo'],
'uploader_url': 'https://blazo.bandcamp.com', 'uploader_url': 'https://blazo.bandcamp.com',
'release_date': '20110727', 'release_date': '20110727',
@@ -291,6 +317,7 @@ class BandcampAlbumIE(BandcampIE): # XXX: Do not subclass from concrete IE
'id': '38097443', 'id': '38097443',
'ext': 'mp3', 'ext': 'mp3',
'title': 'Blazo - Kero One - Keep It Alive (Blazo remix)', 'title': 'Blazo - Kero One - Keep It Alive (Blazo remix)',
'thumbnail': r're:https?://f4\.bcbits\.com/img/.+\.jpg',
'timestamp': 1311757238, 'timestamp': 1311757238,
'upload_date': '20110727', 'upload_date': '20110727',
'uploader': 'Blazo', 'uploader': 'Blazo',
@@ -304,7 +331,6 @@ class BandcampAlbumIE(BandcampIE): # XXX: Do not subclass from concrete IE
'uploader_id': 'blazo', 'uploader_id': 'blazo',
'album_artists': ['Blazo'], 'album_artists': ['Blazo'],
'artists': ['Blazo'], 'artists': ['Blazo'],
'thumbnail': 'https://f4.bcbits.com/img/a1721150828_5.jpg',
'release_timestamp': 1311724800.0, 'release_timestamp': 1311724800.0,
}, },
}, },
plugins/youtube_download/yt_dlp/extractor/bellmedia.py
-91
View File
@@ -1,91 +0,0 @@
from .common import InfoExtractor
class BellMediaIE(InfoExtractor):
_VALID_URL = r'''(?x)https?://(?:www\.)?
(?P<domain>
(?:
ctv|
tsn|
bnn(?:bloomberg)?|
thecomedynetwork|
discovery|
discoveryvelocity|
sciencechannel|
investigationdiscovery|
animalplanet|
bravo|
mtv|
space|
etalk|
marilyn
)\.ca|
(?:much|cp24)\.com
)/.*?(?:\b(?:vid(?:eoid)?|clipId)=|-vid|~|%7E|/(?:episode)?)(?P<id>[0-9]{6,})'''
_TESTS = [{
'url': 'https://www.bnnbloomberg.ca/video/david-cockfield-s-top-picks~1403070',
'md5': '3e5b8e38370741d5089da79161646635',
'info_dict': {
'id': '1403070',
'ext': 'flv',
'title': 'David Cockfield\'s Top Picks',
'description': 'md5:810f7f8c6a83ad5b48677c3f8e5bb2c3',
'upload_date': '20180525',
'timestamp': 1527288600,
'season_id': '73997',
'season': '2018',
'thumbnail': 'http://images2.9c9media.com/image_asset/2018_5_25_baf30cbd-b28d-4a18-9903-4bb8713b00f5_PNG_956x536.jpg',
'tags': [],
'categories': ['ETFs'],
'season_number': 8,
'duration': 272.038,
'series': 'Market Call Tonight',
},
}, {
'url': 'http://www.thecomedynetwork.ca/video/player?vid=923582',
'only_matching': True,
}, {
'url': 'http://www.tsn.ca/video/expectations-high-for-milos-raonic-at-us-open~939549',
'only_matching': True,
}, {
'url': 'http://www.bnn.ca/video/berman-s-call-part-two-viewer-questions~939654',
'only_matching': True,
}, {
'url': 'http://www.ctv.ca/YourMorning/Video/S1E6-Monday-August-29-2016-vid938009',
'only_matching': True,
}, {
'url': 'http://www.much.com/shows/atmidnight/episode948007/tuesday-september-13-2016',
'only_matching': True,
}, {
'url': 'http://www.much.com/shows/the-almost-impossible-gameshow/928979/episode-6',
'only_matching': True,
}, {
'url': 'http://www.ctv.ca/DCs-Legends-of-Tomorrow/Video/S2E11-Turncoat-vid1051430',
'only_matching': True,
}, {
'url': 'http://www.etalk.ca/video?videoid=663455',
'only_matching': True,
}, {
'url': 'https://www.cp24.com/video?clipId=1982548',
'only_matching': True,
}]
_DOMAINS = {
'thecomedynetwork': 'comedy',
'discoveryvelocity': 'discvel',
'sciencechannel': 'discsci',
'investigationdiscovery': 'invdisc',
'animalplanet': 'aniplan',
'etalk': 'ctv',
'bnnbloomberg': 'bnn',
'marilyn': 'ctv_marilyn',
}
def _real_extract(self, url):
domain, video_id = self._match_valid_url(url).groups()
domain = domain.split('.')[0]
return {
'_type': 'url_transparent',
'id': video_id,
'url': f'9c9media:{self._DOMAINS.get(domain, domain)}_web:{video_id}',
'ie_key': 'NineCNineMedia',
}
plugins/youtube_download/yt_dlp/extractor/bet.py
+43 -75
View File
@@ -1,79 +1,47 @@
from .mtv import MTVServicesInfoExtractor from .mtv import MTVServicesBaseIE
from ..utils import unified_strdate
class BetIE(MTVServicesInfoExtractor): class BetIE(MTVServicesBaseIE):
_WORKING = False _VALID_URL = r'https?://(?:www\.)?bet\.com/(?:video-clips|episodes)/(?P<id>[\da-z]{6})'
_VALID_URL = r'https?://(?:www\.)?bet\.com/(?:[^/]+/)+(?P<id>.+?)\.html' _TESTS = [{
_TESTS = [ 'url': 'https://www.bet.com/video-clips/w9mk7v',
{ 'info_dict': {
'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html', 'id': '3022d121-d191-43fd-b5fb-b2c26f335497',
'info_dict': { 'ext': 'mp4',
'id': '07e96bd3-8850-3051-b856-271b457f0ab8', 'display_id': 'w9mk7v',
'display_id': 'in-bet-exclusive-obama-talks-race-and-racism', 'title': 'New Normal',
'ext': 'flv', 'description': 'md5:d7898c124713b4646cecad9d16ff01f3',
'title': 'A Conversation With President Obama', 'duration': 30.08,
'description': 'President Obama urges persistence in confronting racism and bias.', 'series': 'Tyler Perry\'s Sistas',
'duration': 1534, 'season': 'Season 0',
'upload_date': '20141208', 'season_number': 0,
'thumbnail': r're:(?i)^https?://.*\.jpg$', 'episode': 'Episode 0',
'subtitles': { 'episode_number': 0,
'en': 'mincount:2', 'timestamp': 1755269073,
}, 'upload_date': '20250815',
},
'params': {
# rtmp download
'skip_download': True,
},
}, },
{ 'params': {'skip_download': 'm3u8'},
'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html', }, {
'info_dict': { 'url': 'https://www.bet.com/episodes/nmce72/tyler-perry-s-sistas-heavy-is-the-crown-season-9-ep-5',
'id': '9f516bf1-7543-39c4-8076-dd441b459ba9', 'info_dict': {
'display_id': 'justice-for-ferguson-a-community-reacts', 'id': '6427562b-3029-11f0-b405-16fff45bc035',
'ext': 'flv', 'ext': 'mp4',
'title': 'Justice for Ferguson: A Community Reacts', 'display_id': 'nmce72',
'description': 'A BET News special.', 'title': 'Heavy Is the Crown',
'duration': 1696, 'description': 'md5:1ed345d3157a50572d2464afcc7a652a',
'upload_date': '20141125', 'channel': 'BET',
'thumbnail': r're:(?i)^https?://.*\.jpg$', 'duration': 2550.0,
'subtitles': { 'thumbnail': r're:https://images\.paramount\.tech/uri/mgid:arc:imageassetref',
'en': 'mincount:2', 'series': 'Tyler Perry\'s Sistas',
}, 'season': 'Season 9',
}, 'season_number': 9,
'params': { 'episode': 'Episode 5',
# rtmp download 'episode_number': 5,
'skip_download': True, 'timestamp': 1755165600,
}, 'upload_date': '20250814',
'release_timestamp': 1755129600,
'release_date': '20250814',
}, },
] 'params': {'skip_download': 'm3u8'},
'skip': 'Requires provider sign-in',
_FEED_URL = 'http://feeds.mtvnservices.com/od/feed/bet-mrss-player' }]
def _get_feed_query(self, uri):
return {
'uuid': uri,
}
def _extract_mgid(self, webpage):
return self._search_regex(r'data-uri="([^"]+)', webpage, 'mgid')
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
mgid = self._extract_mgid(webpage)
videos_info = self._get_videos_info(mgid)
info_dict = videos_info['entries'][0]
upload_date = unified_strdate(self._html_search_meta('date', webpage))
description = self._html_search_meta('description', webpage)
info_dict.update({
'display_id': display_id,
'description': description,
'upload_date': upload_date,
})
return info_dict
plugins/youtube_download/yt_dlp/extractor/bilibili.py
+134 -19
View File
@@ -304,7 +304,7 @@ class BilibiliBaseIE(InfoExtractor):
class BiliBiliIE(BilibiliBaseIE): class BiliBiliIE(BilibiliBaseIE):
_VALID_URL = r'https?://(?:www\.)?bilibili\.com/(?:video/|festival/[^/?#]+\?(?:[^#]*&)?bvid=)[aAbB][vV](?P<id>[^/?#&]+)' _VALID_URL = r'https?://(?:www\.)?bilibili\.com/(?:video/|festival/[^/?#]+\?(?:[^#]*&)?bvid=)(?P<prefix>[aAbB][vV])(?P<id>[^/?#&]+)'
_TESTS = [{ _TESTS = [{
'url': 'https://www.bilibili.com/video/BV13x41117TL', 'url': 'https://www.bilibili.com/video/BV13x41117TL',
@@ -353,7 +353,7 @@ class BiliBiliIE(BilibiliBaseIE):
'id': 'BV1bK411W797', 'id': 'BV1bK411W797',
'title': '物语中的人物是如何吐槽自己的OP的', 'title': '物语中的人物是如何吐槽自己的OP的',
}, },
'playlist_count': 18, 'playlist_count': 23,
'playlist': [{ 'playlist': [{
'info_dict': { 'info_dict': {
'id': 'BV1bK411W797_p1', 'id': 'BV1bK411W797_p1',
@@ -373,6 +373,7 @@ class BiliBiliIE(BilibiliBaseIE):
'_old_archive_ids': ['bilibili 498159642_part1'], '_old_archive_ids': ['bilibili 498159642_part1'],
}, },
}], }],
'params': {'playlist_items': '2'},
}, { }, {
'note': 'Specific page of Anthology', 'note': 'Specific page of Anthology',
'url': 'https://www.bilibili.com/video/BV1bK411W797?p=1', 'url': 'https://www.bilibili.com/video/BV1bK411W797?p=1',
@@ -562,7 +563,7 @@ class BiliBiliIE(BilibiliBaseIE):
}, },
}], }],
}, { }, {
'note': '301 redirect to bangumi link', 'note': 'redirect from bvid to bangumi link via redirect_url',
'url': 'https://www.bilibili.com/video/BV1TE411f7f1', 'url': 'https://www.bilibili.com/video/BV1TE411f7f1',
'info_dict': { 'info_dict': {
'id': '288525', 'id': '288525',
@@ -579,7 +580,27 @@ class BiliBiliIE(BilibiliBaseIE):
'duration': 1183.957, 'duration': 1183.957,
'timestamp': 1571648124, 'timestamp': 1571648124,
'upload_date': '20191021', 'upload_date': '20191021',
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$', 'thumbnail': r're:https?://.*\.(jpg|jpeg|png)$',
},
}, {
'note': 'redirect from aid to bangumi link via redirect_url',
'url': 'https://www.bilibili.com/video/av114868162141203',
'info_dict': {
'id': '1933368',
'title': 'PV 引爆变革的起点',
'ext': 'mp4',
'duration': 63.139,
'series': '时光代理人',
'series_id': '5183',
'season': '第三季',
'season_number': 4,
'season_id': '105212',
'episode': '引爆变革的起点',
'episode_number': 1,
'episode_id': '1933368',
'timestamp': 1752849001,
'upload_date': '20250718',
'thumbnail': r're:https?://.*\.(jpg|jpeg|png)$',
}, },
}, { }, {
'note': 'video has subtitles, which requires login', 'note': 'video has subtitles, which requires login',
@@ -635,7 +656,7 @@ class BiliBiliIE(BilibiliBaseIE):
}] }]
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id, prefix = self._match_valid_url(url).group('id', 'prefix')
headers = self.geo_verification_headers() headers = self.geo_verification_headers()
webpage, urlh = self._download_webpage_handle(url, video_id, headers=headers) webpage, urlh = self._download_webpage_handle(url, video_id, headers=headers)
if not self._match_valid_url(urlh.url): if not self._match_valid_url(urlh.url):
@@ -643,7 +664,24 @@ class BiliBiliIE(BilibiliBaseIE):
headers['Referer'] = url headers['Referer'] = url
initial_state = self._search_json(r'window\.__INITIAL_STATE__\s*=', webpage, 'initial state', video_id) initial_state = self._search_json(r'window\.__INITIAL_STATE__\s*=', webpage, 'initial state', video_id, default=None)
if not initial_state:
if self._search_json(r'\bwindow\._riskdata_\s*=', webpage, 'risk', video_id, default={}).get('v_voucher'):
raise ExtractorError('You have exceeded the rate limit. Try again later', expected=True)
query = {'platform': 'web'}
prefix = prefix.upper()
if prefix == 'BV':
query['bvid'] = prefix + video_id
elif prefix == 'AV':
query['aid'] = video_id
detail = self._download_json(
'https://api.bilibili.com/x/web-interface/wbi/view/detail', video_id,
note='Downloading redirection URL', errnote='Failed to download redirection URL',
query=self._sign_wbi(query, video_id), headers=headers)
new_url = traverse_obj(detail, ('data', 'View', 'redirect_url', {url_or_none}))
if new_url and BiliBiliBangumiIE.suitable(new_url):
return self.url_result(new_url, BiliBiliBangumiIE)
raise ExtractorError('Unable to extract initial state')
if traverse_obj(initial_state, ('error', 'trueCode')) == -403: if traverse_obj(initial_state, ('error', 'trueCode')) == -403:
self.raise_login_required() self.raise_login_required()
@@ -816,6 +854,26 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
'upload_date': '20111104', 'upload_date': '20111104',
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$', 'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
}, },
}, {
'note': 'new playurlSSRData scheme',
'url': 'https://www.bilibili.com/bangumi/play/ep678060',
'info_dict': {
'id': '678060',
'ext': 'mp4',
'series': '去你家吃饭好吗',
'series_id': '6198',
'season': '第二季',
'season_id': '42542',
'season_number': 2,
'episode': '吴老二：你家大公鸡养不熟，能煮熟吗…',
'episode_id': '678060',
'episode_number': 61,
'title': '一只小九九丫 吴老二：你家大公鸡养不熟，能煮熟吗…',
'duration': 266.123,
'timestamp': 1663315904,
'upload_date': '20220916',
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
},
}, { }, {
'url': 'https://www.bilibili.com/bangumi/play/ep267851', 'url': 'https://www.bilibili.com/bangumi/play/ep267851',
'info_dict': { 'info_dict': {
@@ -879,12 +937,41 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
'Extracting episode', query={'fnval': 12240, 'ep_id': episode_id}, 'Extracting episode', query={'fnval': 12240, 'ep_id': episode_id},
headers=headers)) headers=headers))
premium_only = play_info.get('code') == -10403 # play_info can be structured in at least three different ways, e.g.:
play_info = traverse_obj(play_info, ('result', 'video_info', {dict})) or {} # 1.) play_info['result']['video_info'] and play_info['code']
# 2.) play_info['raw']['data']['video_info'] and play_info['code']
# 3.) play_info['data']['result']['video_info'] and play_info['data']['code']
# So we need to transform any of the above into a common structure
status_code = play_info.get('code')
if 'raw' in play_info:
play_info = play_info['raw']
if 'data' in play_info:
play_info = play_info['data']
if status_code is None:
status_code = play_info.get('code')
if 'result' in play_info:
play_info = play_info['result']
formats = self.extract_formats(play_info) geo_blocked = traverse_obj(play_info, (
if not formats and (premium_only or '成为大会员抢先看' in webpage or '开通大会员观看' in webpage): 'plugins', lambda _, v: v['name'] == 'AreaLimitPanel', 'config', 'is_block', {bool}, any))
self.raise_login_required('This video is for premium members only') premium_only = status_code == -10403
video_info = traverse_obj(play_info, ('video_info', {dict})) or {}
formats = self.extract_formats(video_info)
if not formats:
if geo_blocked:
self.raise_geo_restricted()
elif premium_only or '成为大会员抢先看' in webpage or '开通大会员观看' in webpage:
self.raise_login_required('This video is for premium members only')
if traverse_obj(play_info, ((
('play_check', 'play_detail'), # 'PLAY_PREVIEW' vs 'PLAY_WHOLE' vs 'PLAY_NONE'
'play_video_type', # 'preview' vs 'whole' vs 'none'
), any, {lambda x: x in ('PLAY_PREVIEW', 'preview')})):
self.report_warning(
'Only preview format is available, '
f'you have to become a premium member to access full video. {self._login_hint()}')
bangumi_info = self._download_json( bangumi_info = self._download_json(
'https://api.bilibili.com/pgc/view/web/season', episode_id, 'Get episode details', 'https://api.bilibili.com/pgc/view/web/season', episode_id, 'Get episode details',
@@ -922,7 +1009,7 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
'season': str_or_none(season_title), 'season': str_or_none(season_title),
'season_id': str_or_none(season_id), 'season_id': str_or_none(season_id),
'season_number': season_number, 'season_number': season_number,
'duration': float_or_none(play_info.get('timelength'), scale=1000), 'duration': float_or_none(video_info.get('timelength'), scale=1000),
'subtitles': self.extract_subtitles(episode_id, episode_info.get('cid'), aid=aid), 'subtitles': self.extract_subtitles(episode_id, episode_info.get('cid'), aid=aid),
'__post_extractor': self.extract_comments(aid), '__post_extractor': self.extract_comments(aid),
'http_headers': {'Referer': url}, 'http_headers': {'Referer': url},
@@ -966,6 +1053,7 @@ class BiliBiliBangumiMediaIE(BilibiliBaseIE):
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$', 'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
}, },
}], }],
'params': {'playlist_items': '2'},
}] }]
def _real_extract(self, url): def _real_extract(self, url):
@@ -1021,6 +1109,7 @@ class BiliBiliBangumiSeasonIE(BilibiliBaseIE):
'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$', 'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
}, },
}], }],
'params': {'playlist_items': '2'},
}] }]
def _real_extract(self, url): def _real_extract(self, url):
@@ -1192,6 +1281,26 @@ class BilibiliSpaceVideoIE(BilibiliSpaceBaseIE):
'id': '313580179', 'id': '313580179',
}, },
'playlist_mincount': 92, 'playlist_mincount': 92,
}, {
# Hidden-mode collection
'url': 'https://space.bilibili.com/3669403/video',
'info_dict': {
'id': '3669403',
},
'playlist': [{
'info_dict': {
'_type': 'playlist',
'id': '3669403_3958082',
'title': '合集·直播回放',
'description': '',
'uploader': '月路Yuel',
'uploader_id': '3669403',
'timestamp': int,
'upload_date': str,
'thumbnail': str,
},
}],
'params': {'playlist_items': '7'},
}] }]
def _real_extract(self, url): def _real_extract(self, url):
@@ -1248,10 +1357,16 @@ class BilibiliSpaceVideoIE(BilibiliSpaceBaseIE):
} }
def get_entries(page_data): def get_entries(page_data):
for entry in traverse_obj(page_data, ('list', 'vlist')) or []: for entry in traverse_obj(page_data, ('list', 'vlist', ..., {dict})):
yield self.url_result(f'https://www.bilibili.com/video/{entry["bvid"]}', BiliBiliIE, entry['bvid']) if traverse_obj(entry, ('meta', 'attribute')) == 156:
# hidden-mode collection doesn't show its videos in uploads; extract as playlist instead
yield self.url_result(
f'https://space.bilibili.com/{entry["mid"]}/lists/{entry["meta"]["id"]}?type=season',
BilibiliCollectionListIE, f'{entry["mid"]}_{entry["meta"]["id"]}')
else:
yield self.url_result(f'https://www.bilibili.com/video/{entry["bvid"]}', BiliBiliIE, entry['bvid'])
metadata, paged_list = self._extract_playlist(fetch_page, get_metadata, get_entries) _, paged_list = self._extract_playlist(fetch_page, get_metadata, get_entries)
return self.playlist_result(paged_list, playlist_id) return self.playlist_result(paged_list, playlist_id)
@@ -1285,7 +1400,7 @@ class BilibiliSpaceAudioIE(BilibiliSpaceBaseIE):
for entry in page_data.get('data') or []: for entry in page_data.get('data') or []:
yield self.url_result(f'https://www.bilibili.com/audio/au{entry["id"]}', BilibiliAudioIE, entry['id']) yield self.url_result(f'https://www.bilibili.com/audio/au{entry["id"]}', BilibiliAudioIE, entry['id'])
metadata, paged_list = self._extract_playlist(fetch_page, get_metadata, get_entries) _, paged_list = self._extract_playlist(fetch_page, get_metadata, get_entries)
return self.playlist_result(paged_list, playlist_id) return self.playlist_result(paged_list, playlist_id)
@@ -1785,7 +1900,7 @@ class BilibiliAudioIE(BilibiliAudioBaseIE):
'thumbnail': r're:^https?://.+\.jpg', 'thumbnail': r're:^https?://.+\.jpg',
'timestamp': 1564836614, 'timestamp': 1564836614,
'upload_date': '20190803', 'upload_date': '20190803',
'uploader': 'tsukimi-つきみぐー', 'uploader': '十六夜tsukimiつきみぐ',
'view_count': int, 'view_count': int,
}, },
} }
@@ -1840,10 +1955,10 @@ class BilibiliAudioAlbumIE(BilibiliAudioBaseIE):
'url': 'https://www.bilibili.com/audio/am10624', 'url': 'https://www.bilibili.com/audio/am10624',
'info_dict': { 'info_dict': {
'id': '10624', 'id': '10624',
'title': '每日新曲推荐（每日11:00更新）', 'title': '新曲推荐',
'description': '每天11:00更新，为你推送最新音乐', 'description': '每天11:00更新，为你推送最新音乐',
}, },
'playlist_count': 19, 'playlist_count': 16,
} }
def _real_extract(self, url): def _real_extract(self, url):
plugins/youtube_download/yt_dlp/extractor/bitchute.py
+95 -61
View File
@@ -1,30 +1,32 @@
import functools import functools
import json
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..networking import HEADRequest from ..networking import HEADRequest
from ..networking.exceptions import HTTPError
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
OnDemandPagedList, OnDemandPagedList,
clean_html, clean_html,
extract_attributes, determine_ext,
format_field,
get_element_by_class, get_element_by_class,
get_element_by_id,
get_element_html_by_class,
get_elements_html_by_class, get_elements_html_by_class,
int_or_none, int_or_none,
orderedSet, orderedSet,
parse_count, parse_count,
parse_duration, parse_duration,
traverse_obj, parse_iso8601,
unified_strdate, url_or_none,
urlencode_postdata, urlencode_postdata,
urljoin, urljoin,
) )
from ..utils.traversal import traverse_obj
class BitChuteIE(InfoExtractor): class BitChuteIE(InfoExtractor):
_VALID_URL = r'https?://(?:(?:www|old)\.)?bitchute\.com/(?:video|embed|torrent/[^/]+)/(?P<id>[^/?#&]+)' _VALID_URL = r'https?://(?:(?:www|old)\.)?bitchute\.com/(?:video|embed|torrent/[^/?#]+)/(?P<id>[^/?#&]+)'
_EMBED_REGEX = [rf'<(?:script|iframe)[^>]+\bsrc=(["\'])(?P<url>{_VALID_URL})'] _EMBED_REGEX = [rf'<(?:script|iframe)[^>]+\bsrc=(["\'])(?P<url>{_VALID_URL})']
_TESTS = [{ _TESTS = [{
'url': 'https://www.bitchute.com/video/UGlrF9o9b-Q/', 'url': 'https://www.bitchute.com/video/UGlrF9o9b-Q/',
@@ -34,12 +36,17 @@ class BitChuteIE(InfoExtractor):
'ext': 'mp4', 'ext': 'mp4',
'title': 'This is the first video on #BitChute !', 'title': 'This is the first video on #BitChute !',
'description': 'md5:a0337e7b1fe39e32336974af8173a034', 'description': 'md5:a0337e7b1fe39e32336974af8173a034',
'thumbnail': r're:^https?://.*\.jpg$', 'thumbnail': r're:https?://.+/.+\.jpg$',
'uploader': 'BitChute', 'uploader': 'BitChute',
'upload_date': '20170103', 'upload_date': '20170103',
'uploader_url': 'https://www.bitchute.com/profile/I5NgtHZn9vPj/', 'uploader_url': 'https://www.bitchute.com/profile/I5NgtHZn9vPj/',
'channel': 'BitChute', 'channel': 'BitChute',
'channel_url': 'https://www.bitchute.com/channel/bitchute/', 'channel_url': 'https://www.bitchute.com/channel/bitchute/',
'uploader_id': 'I5NgtHZn9vPj',
'channel_id': '1VBwRfyNcKdX',
'view_count': int,
'duration': 16.0,
'timestamp': 1483425443,
}, },
}, { }, {
# test case: video with different channel and uploader # test case: video with different channel and uploader
@@ -49,13 +56,18 @@ class BitChuteIE(InfoExtractor):
'id': 'Yti_j9A-UZ4', 'id': 'Yti_j9A-UZ4',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Israel at War | Full Measure', 'title': 'Israel at War | Full Measure',
'description': 'md5:38cf7bc6f42da1a877835539111c69ef', 'description': 'md5:e60198b89971966d6030d22b3268f08f',
'thumbnail': r're:^https?://.*\.jpg$', 'thumbnail': r're:https?://.+/.+\.jpg$',
'uploader': 'sharylattkisson', 'uploader': 'sharylattkisson',
'upload_date': '20231106', 'upload_date': '20231106',
'uploader_url': 'https://www.bitchute.com/profile/9K0kUWA9zmd9/', 'uploader_url': 'https://www.bitchute.com/profile/9K0kUWA9zmd9/',
'channel': 'Full Measure with Sharyl Attkisson', 'channel': 'Full Measure with Sharyl Attkisson',
'channel_url': 'https://www.bitchute.com/channel/sharylattkisson/', 'channel_url': 'https://www.bitchute.com/channel/sharylattkisson/',
'uploader_id': '9K0kUWA9zmd9',
'channel_id': 'NpdxoCRv3ZLb',
'view_count': int,
'duration': 554.0,
'timestamp': 1699296106,
}, },
}, { }, {
# video not downloadable in browser, but we can recover it # video not downloadable in browser, but we can recover it
@@ -66,25 +78,21 @@ class BitChuteIE(InfoExtractor):
'ext': 'mp4', 'ext': 'mp4',
'filesize': 71537926, 'filesize': 71537926,
'title': 'STYXHEXENHAMMER666 - Election Fraud, Clinton 2020, EU Armies, and Gun Control', 'title': 'STYXHEXENHAMMER666 - Election Fraud, Clinton 2020, EU Armies, and Gun Control',
'description': 'md5:228ee93bd840a24938f536aeac9cf749', 'description': 'md5:2029c7c212ccd4b040f52bb2d036ef4e',
'thumbnail': r're:^https?://.*\.jpg$', 'thumbnail': r're:https?://.+/.+\.jpg$',
'uploader': 'BitChute', 'uploader': 'BitChute',
'upload_date': '20181113', 'upload_date': '20181113',
'uploader_url': 'https://www.bitchute.com/profile/I5NgtHZn9vPj/', 'uploader_url': 'https://www.bitchute.com/profile/I5NgtHZn9vPj/',
'channel': 'BitChute', 'channel': 'BitChute',
'channel_url': 'https://www.bitchute.com/channel/bitchute/', 'channel_url': 'https://www.bitchute.com/channel/bitchute/',
'uploader_id': 'I5NgtHZn9vPj',
'channel_id': '1VBwRfyNcKdX',
'view_count': int,
'duration': 1701.0,
'tags': ['bitchute'],
'timestamp': 1542130287,
}, },
'params': {'check_formats': None}, 'params': {'check_formats': None},
}, {
# restricted video
'url': 'https://www.bitchute.com/video/WEnQU7XGcTdl/',
'info_dict': {
'id': 'WEnQU7XGcTdl',
'ext': 'mp4',
'title': 'Impartial Truth - Ein Letzter Appell an die Vernunft',
},
'params': {'skip_download': True},
'skip': 'Georestricted in DE',
}, { }, {
'url': 'https://www.bitchute.com/embed/lbb5G1hjPhw/', 'url': 'https://www.bitchute.com/embed/lbb5G1hjPhw/',
'only_matching': True, 'only_matching': True,
@@ -96,11 +104,8 @@ class BitChuteIE(InfoExtractor):
'only_matching': True, 'only_matching': True,
}] }]
_GEO_BYPASS = False _GEO_BYPASS = False
_UPLOADER_URL_TMPL = 'https://www.bitchute.com/profile/%s/'
_HEADERS = { _CHANNEL_URL_TMPL = 'https://www.bitchute.com/channel/%s/'
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.57 Safari/537.36',
'Referer': 'https://www.bitchute.com/',
}
def _check_format(self, video_url, video_id): def _check_format(self, video_url, video_id):
urls = orderedSet( urls = orderedSet(
@@ -112,7 +117,7 @@ class BitChuteIE(InfoExtractor):
for url in urls: for url in urls:
try: try:
response = self._request_webpage( response = self._request_webpage(
HEADRequest(url), video_id=video_id, note=f'Checking {url}', headers=self._HEADERS) HEADRequest(url), video_id=video_id, note=f'Checking {url}')
except ExtractorError as e: except ExtractorError as e:
self.to_screen(f'{video_id}: URL is invalid, skipping: {e.cause}') self.to_screen(f'{video_id}: URL is invalid, skipping: {e.cause}')
continue continue
@@ -121,54 +126,79 @@ class BitChuteIE(InfoExtractor):
'filesize': int_or_none(response.headers.get('Content-Length')), 'filesize': int_or_none(response.headers.get('Content-Length')),
} }
def _raise_if_restricted(self, webpage): def _call_api(self, endpoint, data, display_id, fatal=True):
page_title = clean_html(get_element_by_class('page-title', webpage)) or '' note = endpoint.rpartition('/')[2]
if re.fullmatch(r'(?:Channel|Video) Restricted', page_title): try:
reason = clean_html(get_element_by_id('page-detail', webpage)) or page_title return self._download_json(
self.raise_geo_restricted(reason) f'https://api.bitchute.com/api/beta/{endpoint}', display_id,
f'Downloading {note} API JSON', f'Unable to download {note} API JSON',
@staticmethod data=json.dumps(data).encode(),
def _make_url(html): headers={
path = extract_attributes(get_element_html_by_class('spa', html) or '').get('href') 'Accept': 'application/json',
return urljoin('https://www.bitchute.com', path) 'Content-Type': 'application/json',
})
except ExtractorError as e:
if isinstance(e.cause, HTTPError) and e.cause.status == 403:
errors = '. '.join(traverse_obj(e.cause.response.read().decode(), (
{json.loads}, 'errors', lambda _, v: v['context'] == 'reason', 'message', {str})))
if errors and 'location' in errors:
# Can always be fatal since the video/media call will reach this code first
self.raise_geo_restricted(errors)
if fatal:
raise
self.report_warning(e.msg)
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
webpage = self._download_webpage( data = {'video_id': video_id}
f'https://old.bitchute.com/video/{video_id}', video_id, headers=self._HEADERS) media_url = self._call_api('video/media', data, video_id)['media_url']
self._raise_if_restricted(webpage)
publish_date = clean_html(get_element_by_class('video-publish-date', webpage))
entries = self._parse_html5_media_entries(url, webpage, video_id)
formats = [] formats = []
for format_ in traverse_obj(entries, (0, 'formats', ...)): if determine_ext(media_url) == 'm3u8':
formats.extend(
self._extract_m3u8_formats(media_url, video_id, 'mp4', m3u8_id='hls', live=True))
else:
if self.get_param('check_formats') is not False: if self.get_param('check_formats') is not False:
format_.update(self._check_format(format_.pop('url'), video_id) or {}) if fmt := self._check_format(media_url, video_id):
if 'url' not in format_: formats.append(fmt)
continue else:
formats.append(format_) formats.append({'url': media_url})
if not formats: if not formats:
self.raise_no_formats( self.raise_no_formats(
'Video is unavailable. Please make sure this video is playable in the browser ' 'Video is unavailable. Please make sure this video is playable in the browser '
'before reporting this issue.', expected=True, video_id=video_id) 'before reporting this issue.', expected=True, video_id=video_id)
details = get_element_by_class('details', webpage) or '' video = self._call_api('video', data, video_id, fatal=False)
uploader_html = get_element_html_by_class('creator', details) or '' channel = None
channel_html = get_element_html_by_class('name', details) or '' if channel_id := traverse_obj(video, ('channel', 'channel_id', {str})):
channel = self._call_api('channel', {'channel_id': channel_id}, video_id, fatal=False)
return { return {
**traverse_obj(video, {
'title': ('video_name', {str}),
'description': ('description', {str}),
'thumbnail': ('thumbnail_url', {url_or_none}),
'channel': ('channel', 'channel_name', {str}),
'channel_id': ('channel', 'channel_id', {str}),
'channel_url': ('channel', 'channel_url', {urljoin('https://www.bitchute.com/')}),
'uploader_id': ('profile_id', {str}),
'uploader_url': ('profile_id', {format_field(template=self._UPLOADER_URL_TMPL)}, filter),
'timestamp': ('date_published', {parse_iso8601}),
'duration': ('duration', {parse_duration}),
'tags': ('hashtags', ..., {str}, filter, all, filter),
'view_count': ('view_count', {int_or_none}),
'is_live': ('state_id', {lambda x: x == 'live'}),
}),
**traverse_obj(channel, {
'channel': ('channel_name', {str}),
'channel_id': ('channel_id', {str}),
'channel_url': ('url_slug', {format_field(template=self._CHANNEL_URL_TMPL)}, filter),
'uploader': ('profile_name', {str}),
'uploader_id': ('profile_id', {str}),
'uploader_url': ('profile_id', {format_field(template=self._UPLOADER_URL_TMPL)}, filter),
}),
'id': video_id, 'id': video_id,
'title': self._html_extract_title(webpage) or self._og_search_title(webpage),
'description': self._og_search_description(webpage, default=None),
'thumbnail': self._og_search_thumbnail(webpage),
'uploader': clean_html(uploader_html),
'uploader_url': self._make_url(uploader_html),
'channel': clean_html(channel_html),
'channel_url': self._make_url(channel_html),
'upload_date': unified_strdate(self._search_regex(
r'at \d+:\d+ UTC on (.+?)\.', publish_date, 'upload date', fatal=False)),
'formats': formats, 'formats': formats,
} }
@@ -190,7 +220,7 @@ class BitChuteChannelIE(InfoExtractor):
'ext': 'mp4', 'ext': 'mp4',
'title': 'This is the first video on #BitChute !', 'title': 'This is the first video on #BitChute !',
'description': 'md5:a0337e7b1fe39e32336974af8173a034', 'description': 'md5:a0337e7b1fe39e32336974af8173a034',
'thumbnail': r're:^https?://.*\.jpg$', 'thumbnail': r're:https?://.+/.+\.jpg$',
'uploader': 'BitChute', 'uploader': 'BitChute',
'upload_date': '20170103', 'upload_date': '20170103',
'uploader_url': 'https://www.bitchute.com/profile/I5NgtHZn9vPj/', 'uploader_url': 'https://www.bitchute.com/profile/I5NgtHZn9vPj/',
@@ -198,6 +228,9 @@ class BitChuteChannelIE(InfoExtractor):
'channel_url': 'https://www.bitchute.com/channel/bitchute/', 'channel_url': 'https://www.bitchute.com/channel/bitchute/',
'duration': 16, 'duration': 16,
'view_count': int, 'view_count': int,
'uploader_id': 'I5NgtHZn9vPj',
'channel_id': '1VBwRfyNcKdX',
'timestamp': 1483425443,
}, },
}, },
], ],
@@ -213,6 +246,7 @@ class BitChuteChannelIE(InfoExtractor):
'title': 'Bruce MacDonald and "The Light of Darkness"', 'title': 'Bruce MacDonald and "The Light of Darkness"',
'description': 'md5:747724ef404eebdfc04277714f81863e', 'description': 'md5:747724ef404eebdfc04277714f81863e',
}, },
'skip': '404 Not Found',
}, { }, {
'url': 'https://old.bitchute.com/playlist/wV9Imujxasw9/', 'url': 'https://old.bitchute.com/playlist/wV9Imujxasw9/',
'only_matching': True, 'only_matching': True,
plugins/youtube_download/yt_dlp/extractor/blackboardcollaborate.py
+131 -15
View File
@@ -1,16 +1,27 @@
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import parse_iso8601 from ..utils import (
UnsupportedError,
float_or_none,
int_or_none,
join_nonempty,
jwt_decode_hs256,
mimetype2ext,
parse_iso8601,
parse_qs,
url_or_none,
)
from ..utils.traversal import traverse_obj
class BlackboardCollaborateIE(InfoExtractor): class BlackboardCollaborateIE(InfoExtractor):
_VALID_URL = r'''(?x) _VALID_URL = r'''(?x)
https?:// https?://
(?P<region>[a-z-]+)\.bbcollab\.com/ (?P<region>[a-z]+)(?:-lti)?\.bbcollab\.com/
(?: (?:
collab/ui/session/playback/load| collab/ui/session/playback/load|
recording recording
)/ )/
(?P<id>[^/]+)''' (?P<id>[^/?#]+)'''
_TESTS = [ _TESTS = [
{ {
'url': 'https://us-lti.bbcollab.com/collab/ui/session/playback/load/0a633b6a88824deb8c918f470b22b256', 'url': 'https://us-lti.bbcollab.com/collab/ui/session/playback/load/0a633b6a88824deb8c918f470b22b256',
@@ -19,9 +30,55 @@ class BlackboardCollaborateIE(InfoExtractor):
'id': '0a633b6a88824deb8c918f470b22b256', 'id': '0a633b6a88824deb8c918f470b22b256',
'title': 'HESI A2 Information Session - Thursday, May 6, 2021 - recording_1', 'title': 'HESI A2 Information Session - Thursday, May 6, 2021 - recording_1',
'ext': 'mp4', 'ext': 'mp4',
'duration': 1896000, 'duration': 1896,
'timestamp': 1620331399, 'timestamp': 1620333295,
'upload_date': '20210506', 'upload_date': '20210506',
'subtitles': {
'live_chat': 'mincount:1',
},
},
},
{
'url': 'https://eu.bbcollab.com/collab/ui/session/playback/load/4bde2dee104f40289a10f8e554270600',
'md5': '108db6a8f83dcb0c2a07793649581865',
'info_dict': {
'id': '4bde2dee104f40289a10f8e554270600',
'title': 'Meeting - Azerbaycanca erize formasi',
'ext': 'mp4',
'duration': 880,
'timestamp': 1671176868,
'upload_date': '20221216',
},
},
{
'url': 'https://eu.bbcollab.com/recording/f83be390ecff46c0bf7dccb9dddcf5f6',
'md5': 'e3b0b88ddf7847eae4b4c0e2d40b83a5',
'info_dict': {
'id': 'f83be390ecff46c0bf7dccb9dddcf5f6',
'title': 'Keynote lecture by Laura Carvalho - recording_1',
'ext': 'mp4',
'duration': 5506,
'timestamp': 1662721705,
'upload_date': '20220909',
'subtitles': {
'live_chat': 'mincount:1',
},
},
},
{
'url': 'https://eu.bbcollab.com/recording/c3e1e7c9e83d4cd9981c93c74888d496',
'md5': 'fdb2d8c43d66fbc0b0b74ef5e604eb1f',
'info_dict': {
'id': 'c3e1e7c9e83d4cd9981c93c74888d496',
'title': 'International Ally User Group - recording_18',
'ext': 'mp4',
'duration': 3479,
'timestamp': 1721919621,
'upload_date': '20240725',
'subtitles': {
'en': 'mincount:1',
'live_chat': 'mincount:1',
},
}, },
}, },
{ {
@@ -42,22 +99,81 @@ class BlackboardCollaborateIE(InfoExtractor):
}, },
] ]
def _call_api(self, region, video_id, path=None, token=None, note=None, fatal=False):
# Ref: https://github.com/blackboard/BBDN-Collab-Postman-REST
return self._download_json(
join_nonempty(f'https://{region}.bbcollab.com/collab/api/csa/recordings', video_id, path, delim='/'),
video_id, note or 'Downloading JSON metadata', fatal=fatal,
headers={'Authorization': f'Bearer {token}'} if token else None)
def _real_extract(self, url): def _real_extract(self, url):
mobj = self._match_valid_url(url) mobj = self._match_valid_url(url)
region = mobj.group('region') region = mobj.group('region')
video_id = mobj.group('id') video_id = mobj.group('id')
info = self._download_json( token = parse_qs(url).get('authToken', [None])[-1]
f'https://{region}.bbcollab.com/collab/api/csa/recordings/{video_id}/data', video_id)
duration = info.get('duration') video_info = self._call_api(region, video_id, path='data/secure', token=token, note='Trying auth token')
title = info['name'] if video_info:
upload_date = info.get('created') video_extra = self._call_api(region, video_id, token=token, note='Retrieving extra attributes')
streams = info['streams'] else:
formats = [{'format_id': k, 'url': url} for k, url in streams.items()] video_info = self._call_api(region, video_id, path='data', note='Trying fallback', fatal=True)
video_extra = {}
formats = traverse_obj(video_info, ('extStreams', lambda _, v: url_or_none(v['streamUrl']), {
'url': 'streamUrl',
'ext': ('contentType', {mimetype2ext}),
'aspect_ratio': ('aspectRatio', {float_or_none}),
}))
if filesize := traverse_obj(video_extra, ('storageSize', {int_or_none})):
for fmt in formats:
fmt['filesize'] = filesize
subtitles = {}
for subs in traverse_obj(video_info, ('subtitles', lambda _, v: url_or_none(v['url']))):
subtitles.setdefault(subs.get('lang') or 'und', []).append({
'name': traverse_obj(subs, ('label', {str})),
'url': subs['url'],
})
for live_chat_url in traverse_obj(video_info, ('chats', ..., 'url', {url_or_none})):
subtitles.setdefault('live_chat', []).append({'url': live_chat_url})
return { return {
'duration': duration, **traverse_obj(video_info, {
'title': ('name', {str}),
'timestamp': ('created', {parse_iso8601}),
'duration': ('duration', {int_or_none(scale=1000)}),
}),
'formats': formats, 'formats': formats,
'id': video_id, 'id': video_id,
'timestamp': parse_iso8601(upload_date), 'subtitles': subtitles,
'title': title,
} }
class BlackboardCollaborateLaunchIE(InfoExtractor):
_VALID_URL = r'https?://[a-z]+\.bbcollab\.com/launch/(?P<id>[^/?#]+)'
_TESTS = [
{
'url': 'https://au.bbcollab.com/launch/eyJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJiYkNvbGxhYkFwaSIsInN1YiI6ImJiQ29sbGFiQXBpIiwiZXhwIjoxNzQwNDE2NDgzLCJpYXQiOjE3NDA0MTYxODMsInJlc291cmNlQWNjZXNzVGlja2V0Ijp7InJlc291cmNlSWQiOiI3MzI4YzRjZTNmM2U0ZTcwYmY3MTY3N2RkZTgzMzk2NSIsImNvbnN1bWVySWQiOiJhM2Q3NGM0Y2QyZGU0MGJmODFkMjFlODNlMmEzNzM5MCIsInR5cGUiOiJSRUNPUkRJTkciLCJyZXN0cmljdGlvbiI6eyJ0eXBlIjoiVElNRSIsImV4cGlyYXRpb25Ib3VycyI6MCwiZXhwaXJhdGlvbk1pbnV0ZXMiOjUsIm1heFJlcXVlc3RzIjotMX0sImRpc3Bvc2l0aW9uIjoiTEFVTkNIIiwibGF1bmNoVHlwZSI6bnVsbCwibGF1bmNoQ29tcG9uZW50IjpudWxsLCJsYXVuY2hQYXJhbUtleSI6bnVsbH19.xuELw4EafEwUMoYcCHidGn4Tw9O1QCbYHzYGJUl0kKk',
'only_matching': True,
},
{
'url': 'https://us.bbcollab.com/launch/eyJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJiYkNvbGxhYkFwaSIsInN1YiI6ImJiQ29sbGFiQXBpIiwiZXhwIjoxNjk0NDgxOTc3LCJpYXQiOjE2OTQ0ODE2NzcsInJlc291cmNlQWNjZXNzVGlja2V0Ijp7InJlc291cmNlSWQiOiI3YWU0MTFhNTU3NjU0OWFiOTZlYjVmMTM1YmY3MWU5MCIsImNvbnN1bWVySWQiOiJBRUU2MEI4MDI2QzM3ODU2RjMwMzNEN0ZEOTQzMTFFNSIsInR5cGUiOiJSRUNPUkRJTkciLCJyZXN0cmljdGlvbiI6eyJ0eXBlIjoiVElNRSIsImV4cGlyYXRpb25Ib3VycyI6MCwiZXhwaXJhdGlvbk1pbnV0ZXMiOjUsIm1heFJlcXVlc3RzIjotMX0sImRpc3Bvc2l0aW9uIjoiTEFVTkNIIiwibGF1bmNoVHlwZSI6bnVsbCwibGF1bmNoQ29tcG9uZW50IjpudWxsLCJsYXVuY2hQYXJhbUtleSI6bnVsbH19.yOhRZNaIjXYoMYMpcTzgjZJCnIFaYf2cAzbco8OAxlY',
'only_matching': True,
},
{
'url': 'https://eu.bbcollab.com/launch/eyJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJiYkNvbGxhYkFwaSIsInN1YiI6ImJiQ29sbGFiQXBpIiwiZXhwIjoxNzUyNjgyODYwLCJpYXQiOjE3NTI2ODI1NjAsInJlc291cmNlQWNjZXNzVGlja2V0Ijp7InJlc291cmNlSWQiOiI4MjQzYjFiODg2Nzk0NTZkYjkwN2NmNDZmZmE1MmFhZiIsImNvbnN1bWVySWQiOiI5ZTY4NzYwZWJiNzM0MzRiYWY3NTQyZjA1YmJkOTMzMCIsInR5cGUiOiJSRUNPUkRJTkciLCJyZXN0cmljdGlvbiI6eyJ0eXBlIjoiVElNRSIsImV4cGlyYXRpb25Ib3VycyI6MCwiZXhwaXJhdGlvbk1pbnV0ZXMiOjUsIm1heFJlcXVlc3RzIjotMX0sImRpc3Bvc2l0aW9uIjoiTEFVTkNIIiwibGF1bmNoVHlwZSI6bnVsbCwibGF1bmNoQ29tcG9uZW50IjpudWxsLCJsYXVuY2hQYXJhbUtleSI6bnVsbH19.Xj4ymojYLwZ1vKPKZ-KxjpqQvFXoJekjRaG0npngwWs',
'only_matching': True,
},
]
def _real_extract(self, url):
token = self._match_id(url)
video_id = jwt_decode_hs256(token)['resourceAccessTicket']['resourceId']
redirect_url = self._request_webpage(url, video_id).url
if self.suitable(redirect_url):
raise UnsupportedError(redirect_url)
return self.url_result(redirect_url, BlackboardCollaborateIE, video_id)
plugins/youtube_download/yt_dlp/extractor/blogger.py
+12 -1
View File
@@ -19,8 +19,19 @@ class BloggerIE(InfoExtractor):
'id': 'BLOGGER-video-3c740e3a49197e16-796', 'id': 'BLOGGER-video-3c740e3a49197e16-796',
'title': 'BLOGGER-video-3c740e3a49197e16-796', 'title': 'BLOGGER-video-3c740e3a49197e16-796',
'ext': 'mp4', 'ext': 'mp4',
'thumbnail': r're:^https?://.*',
'duration': 76.068, 'duration': 76.068,
'thumbnail': r're:https?://i9\.ytimg\.com/vi_blogger/.+',
},
}]
_WEBPAGE_TESTS = [{
'url': 'https://blog.tomeuvizoso.net/2019/01/a-panfrost-milestone.html',
'md5': 'f1bc19b6ea1b0fd1d81e84ca9ec467ac',
'info_dict': {
'id': 'BLOGGER-video-3c740e3a49197e16-12203',
'ext': 'mp4',
'title': 'BLOGGER-video-3c740e3a49197e16-12203',
'duration': 76.068,
'thumbnail': r're:https?://i9\.ytimg\.com/vi_blogger/.+',
}, },
}] }]
plugins/youtube_download/yt_dlp/extractor/brainpop.py
+1 -1
View File
@@ -174,7 +174,7 @@ class BrainPOPLegacyBaseIE(BrainPOPBaseIE):
} }
def _real_extract(self, url): def _real_extract(self, url):
slug, display_id = self._match_valid_url(url).group('slug', 'id') display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id) webpage = self._download_webpage(url, display_id)
topic_data = self._search_json( topic_data = self._search_json(
r'var\s+content\s*=\s*', webpage, 'content data', r'var\s+content\s*=\s*', webpage, 'content data',
plugins/youtube_download/yt_dlp/extractor/bravotv.py
-188
View File
@@ -1,188 +0,0 @@
from .adobepass import AdobePassIE
from ..networking import HEADRequest
from ..utils import (
extract_attributes,
float_or_none,
get_element_html_by_class,
int_or_none,
merge_dicts,
parse_age_limit,
remove_end,
str_or_none,
traverse_obj,
unescapeHTML,
unified_timestamp,
update_url_query,
url_or_none,
)
class BravoTVIE(AdobePassIE):
_VALID_URL = r'https?://(?:www\.)?(?P<site>bravotv|oxygen)\.com/(?:[^/]+/)+(?P<id>[^/?#]+)'
_TESTS = [{
'url': 'https://www.bravotv.com/top-chef/season-16/episode-15/videos/the-top-chef-season-16-winner-is',
'info_dict': {
'id': '3923059',
'ext': 'mp4',
'title': 'The Top Chef Season 16 Winner Is...',
'description': 'Find out who takes the title of Top Chef!',
'upload_date': '20190314',
'timestamp': 1552591860,
'season_number': 16,
'episode_number': 15,
'series': 'Top Chef',
'episode': 'The Top Chef Season 16 Winner Is...',
'duration': 190.357,
'season': 'Season 16',
'thumbnail': r're:^https://.+\.jpg',
},
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://www.bravotv.com/top-chef/season-20/episode-1/london-calling',
'info_dict': {
'id': '9000234570',
'ext': 'mp4',
'title': 'London Calling',
'description': 'md5:5af95a8cbac1856bd10e7562f86bb759',
'upload_date': '20230310',
'timestamp': 1678410000,
'season_number': 20,
'episode_number': 1,
'series': 'Top Chef',
'episode': 'London Calling',
'duration': 3266.03,
'season': 'Season 20',
'chapters': 'count:7',
'thumbnail': r're:^https://.+\.jpg',
'age_limit': 14,
},
'params': {'skip_download': 'm3u8'},
'skip': 'This video requires AdobePass MSO credentials',
}, {
'url': 'https://www.oxygen.com/in-ice-cold-blood/season-1/closing-night',
'info_dict': {
'id': '3692045',
'ext': 'mp4',
'title': 'Closing Night',
'description': 'md5:3170065c5c2f19548d72a4cbc254af63',
'upload_date': '20180401',
'timestamp': 1522623600,
'season_number': 1,
'episode_number': 1,
'series': 'In Ice Cold Blood',
'episode': 'Closing Night',
'duration': 2629.051,
'season': 'Season 1',
'chapters': 'count:6',
'thumbnail': r're:^https://.+\.jpg',
'age_limit': 14,
},
'params': {'skip_download': 'm3u8'},
'skip': 'This video requires AdobePass MSO credentials',
}, {
'url': 'https://www.oxygen.com/in-ice-cold-blood/season-2/episode-16/videos/handling-the-horwitz-house-after-the-murder-season-2',
'info_dict': {
'id': '3974019',
'ext': 'mp4',
'title': '\'Handling The Horwitz House After The Murder (Season 2, Episode 16)',
'description': 'md5:f9d638dd6946a1c1c0533a9c6100eae5',
'upload_date': '20190617',
'timestamp': 1560790800,
'season_number': 2,
'episode_number': 16,
'series': 'In Ice Cold Blood',
'episode': '\'Handling The Horwitz House After The Murder (Season 2, Episode 16)',
'duration': 68.235,
'season': 'Season 2',
'thumbnail': r're:^https://.+\.jpg',
'age_limit': 14,
},
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://www.bravotv.com/below-deck/season-3/ep-14-reunion-part-1',
'only_matching': True,
}]
def _real_extract(self, url):
site, display_id = self._match_valid_url(url).group('site', 'id')
webpage = self._download_webpage(url, display_id)
settings = self._search_json(
r'<script[^>]+data-drupal-selector="drupal-settings-json"[^>]*>', webpage, 'settings', display_id)
tve = extract_attributes(get_element_html_by_class('tve-video-deck-app', webpage) or '')
query = {
'manifest': 'm3u',
'formats': 'm3u,mpeg4',
}
if tve:
account_pid = tve.get('data-mpx-media-account-pid') or 'HNK2IC'
account_id = tve['data-mpx-media-account-id']
metadata = self._parse_json(
tve.get('data-normalized-video', ''), display_id, fatal=False, transform_source=unescapeHTML)
video_id = tve.get('data-guid') or metadata['guid']
if tve.get('data-entitlement') == 'auth':
auth = traverse_obj(settings, ('tve_adobe_auth', {dict})) or {}
site = remove_end(site, 'tv')
release_pid = tve['data-release-pid']
resource = self._get_mvpd_resource(
tve.get('data-adobe-pass-resource-id') or auth.get('adobePassResourceId') or site,
tve['data-title'], release_pid, tve.get('data-rating'))
query.update({
'switch': 'HLSServiceSecure',
'auth': self._extract_mvpd_auth(
url, release_pid, auth.get('adobePassRequestorId') or site, resource),
})
else:
ls_playlist = traverse_obj(settings, ('ls_playlist', ..., {dict}), get_all=False) or {}
account_pid = ls_playlist.get('mpxMediaAccountPid') or 'PHSl-B'
account_id = ls_playlist['mpxMediaAccountId']
video_id = ls_playlist['defaultGuid']
metadata = traverse_obj(
ls_playlist, ('videos', lambda _, v: v['guid'] == video_id, {dict}), get_all=False)
tp_url = f'https://link.theplatform.com/s/{account_pid}/media/guid/{account_id}/{video_id}'
tp_metadata = self._download_json(
update_url_query(tp_url, {'format': 'preview'}), video_id, fatal=False)
chapters = traverse_obj(tp_metadata, ('chapters', ..., {
'start_time': ('startTime', {float_or_none(scale=1000)}),
'end_time': ('endTime', {float_or_none(scale=1000)}),
}))
# prune pointless single chapters that span the entire duration from short videos
if len(chapters) == 1 and not traverse_obj(chapters, (0, 'end_time')):
chapters = None
m3u8_url = self._request_webpage(HEADRequest(
update_url_query(f'{tp_url}/stream.m3u8', query)), video_id, 'Checking m3u8 URL').url
if 'mpeg_cenc' in m3u8_url:
self.report_drm(video_id)
formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
return {
'id': video_id,
'formats': formats,
'subtitles': subtitles,
'chapters': chapters,
**merge_dicts(traverse_obj(tp_metadata, {
'title': 'title',
'description': 'description',
'duration': ('duration', {float_or_none(scale=1000)}),
'timestamp': ('pubDate', {float_or_none(scale=1000)}),
'season_number': (('pl1$seasonNumber', 'nbcu$seasonNumber'), {int_or_none}),
'episode_number': (('pl1$episodeNumber', 'nbcu$episodeNumber'), {int_or_none}),
'series': (('pl1$show', 'nbcu$show'), (None, ...), {str}),
'episode': (('title', 'pl1$episodeNumber', 'nbcu$episodeNumber'), {str_or_none}),
'age_limit': ('ratings', ..., 'rating', {parse_age_limit}),
}, get_all=False), traverse_obj(metadata, {
'title': 'title',
'description': 'description',
'duration': ('durationInSeconds', {int_or_none}),
'timestamp': ('airDate', {unified_timestamp}),
'thumbnail': ('thumbnailUrl', {url_or_none}),
'season_number': ('seasonNumber', {int_or_none}),
'episode_number': ('episodeNumber', {int_or_none}),
'episode': 'episodeTitle',
'series': 'show',
})),
}
plugins/youtube_download/yt_dlp/extractor/brightcove.py
+17 -14
View File
@@ -495,8 +495,6 @@ class BrightcoveLegacyIE(InfoExtractor):
class BrightcoveNewBaseIE(AdobePassIE): class BrightcoveNewBaseIE(AdobePassIE):
def _parse_brightcove_metadata(self, json_data, video_id, headers={}): def _parse_brightcove_metadata(self, json_data, video_id, headers={}):
title = json_data['name'].strip()
formats, subtitles = [], {} formats, subtitles = [], {}
sources = json_data.get('sources') or [] sources = json_data.get('sources') or []
for source in sources: for source in sources:
@@ -600,16 +598,18 @@ class BrightcoveNewBaseIE(AdobePassIE):
return { return {
'id': video_id, 'id': video_id,
'title': title,
'description': clean_html(json_data.get('description')),
'thumbnails': thumbnails, 'thumbnails': thumbnails,
'duration': duration, 'duration': duration,
'timestamp': parse_iso8601(json_data.get('published_at')),
'uploader_id': json_data.get('account_id'),
'formats': formats, 'formats': formats,
'subtitles': subtitles, 'subtitles': subtitles,
'tags': json_data.get('tags', []),
'is_live': is_live, 'is_live': is_live,
**traverse_obj(json_data, {
'title': ('name', {clean_html}),
'description': ('description', {clean_html}),
'tags': ('tags', ..., {str}, filter, all, filter),
'timestamp': ('published_at', {parse_iso8601}),
'uploader_id': ('account_id', {str}),
}),
} }
@@ -645,10 +645,7 @@ class BrightcoveNewIE(BrightcoveNewBaseIE):
'uploader_id': '4036320279001', 'uploader_id': '4036320279001',
'formats': 'mincount:39', 'formats': 'mincount:39',
}, },
'params': { 'skip': '404 Not Found',
# m3u8 download
'skip_download': True,
},
}, { }, {
# playlist stream # playlist stream
'url': 'https://players.brightcove.net/1752604059001/S13cJdUBz_default/index.html?playlistId=5718313430001', 'url': 'https://players.brightcove.net/1752604059001/S13cJdUBz_default/index.html?playlistId=5718313430001',
@@ -709,7 +706,6 @@ class BrightcoveNewIE(BrightcoveNewBaseIE):
'ext': 'mp4', 'ext': 'mp4',
'title': 'TGD_01-032_5', 'title': 'TGD_01-032_5',
'thumbnail': r're:^https?://.*\.jpg$', 'thumbnail': r're:^https?://.*\.jpg$',
'tags': [],
'timestamp': 1646078943, 'timestamp': 1646078943,
'uploader_id': '1569565978001', 'uploader_id': '1569565978001',
'upload_date': '20220228', 'upload_date': '20220228',
@@ -721,7 +717,6 @@ class BrightcoveNewIE(BrightcoveNewBaseIE):
'ext': 'mp4', 'ext': 'mp4',
'title': 'TGD 01-087 (Airs 05.25.22)_Segment 5', 'title': 'TGD 01-087 (Airs 05.25.22)_Segment 5',
'thumbnail': r're:^https?://.*\.jpg$', 'thumbnail': r're:^https?://.*\.jpg$',
'tags': [],
'timestamp': 1651604591, 'timestamp': 1651604591,
'uploader_id': '1569565978001', 'uploader_id': '1569565978001',
'upload_date': '20220503', 'upload_date': '20220503',
@@ -923,10 +918,18 @@ class BrightcoveNewIE(BrightcoveNewBaseIE):
errors = json_data.get('errors') errors = json_data.get('errors')
if errors and errors[0].get('error_subcode') == 'TVE_AUTH': if errors and errors[0].get('error_subcode') == 'TVE_AUTH':
custom_fields = json_data['custom_fields'] custom_fields = json_data['custom_fields']
missing_fields = ', '.join(
key for key in ('source_url', 'software_statement') if not smuggled_data.get(key))
if missing_fields:
raise ExtractorError(
f'Missing fields in smuggled data: {missing_fields}. '
f'This video can be only extracted from the webpage where it is embedded. '
f'Pass the URL of the embedding webpage instead of the Brightcove URL', expected=True)
tve_token = self._extract_mvpd_auth( tve_token = self._extract_mvpd_auth(
smuggled_data['source_url'], video_id, smuggled_data['source_url'], video_id,
custom_fields['bcadobepassrequestorid'], custom_fields['bcadobepassrequestorid'],
custom_fields['bcadobepassresourceid']) custom_fields['bcadobepassresourceid'],
smuggled_data['software_statement'])
json_data = self._download_json( json_data = self._download_json(
api_url, video_id, headers={ api_url, video_id, headers={
'Accept': f'application/json;pk={policy_key}', 'Accept': f'application/json;pk={policy_key}',
plugins/youtube_download/yt_dlp/extractor/btvplus.py
+73
View File
@@ -0,0 +1,73 @@
from .common import InfoExtractor
from ..utils import (
bug_reports_message,
clean_html,
get_element_by_class,
js_to_json,
mimetype2ext,
strip_or_none,
url_or_none,
urljoin,
)
from ..utils.traversal import traverse_obj
class BTVPlusIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?btvplus\.bg/produkt/(?:predavaniya|seriali|novini)/(?P<id>\d+)'
_TESTS = [{
'url': 'https://btvplus.bg/produkt/predavaniya/67271/btv-reporterite/btv-reporterite-12-07-2025-g',
'info_dict': {
'ext': 'mp4',
'id': '67271',
'title': 'bTV Репортерите - 12.07.2025 г.',
'thumbnail': 'https://cdn.btv.bg/media/images/940x529/Jul2025/2113606319.jpg',
},
}, {
'url': 'https://btvplus.bg/produkt/seriali/66942/sezon-2/plen-sezon-2-epizod-55',
'info_dict': {
'ext': 'mp4',
'id': '66942',
'title': 'Плен - сезон 2, епизод 55',
'thumbnail': 'https://cdn.btv.bg/media/images/940x529/Jun2025/2113595104.jpg',
},
}, {
'url': 'https://btvplus.bg/produkt/novini/67270/btv-novinite-centralna-emisija-12-07-2025',
'only_matching': True,
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
player_url = self._search_regex(
r'var\s+videoUrl\s*=\s*[\'"]([^\'"]+)[\'"]',
webpage, 'player URL')
player_config = self._download_json(
urljoin('https://btvplus.bg', player_url), video_id)['config']
videojs_data = self._search_json(
r'videojs\(["\'][^"\']+["\'],', player_config, 'videojs data',
video_id, transform_source=js_to_json)
formats = []
subtitles = {}
for src in traverse_obj(videojs_data, ('sources', lambda _, v: url_or_none(v['src']))):
ext = mimetype2ext(src.get('type'))
if ext == 'm3u8':
fmts, subs = self._extract_m3u8_formats_and_subtitles(
src['src'], video_id, 'mp4', m3u8_id='hls', fatal=False)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
else:
self.report_warning(f'Unknown format type {ext}{bug_reports_message()}')
return {
'id': video_id,
'formats': formats,
'subtitles': subtitles,
'title': (
strip_or_none(self._og_search_title(webpage, default=None))
or clean_html(get_element_by_class('product-title', webpage))),
'thumbnail': self._og_search_thumbnail(webpage, default=None),
'description': self._og_search_description(webpage, default=None),
}
plugins/youtube_download/yt_dlp/extractor/cartoonnetwork.py
-59
View File
@@ -1,59 +0,0 @@
from .turner import TurnerBaseIE
from ..utils import int_or_none
class CartoonNetworkIE(TurnerBaseIE):
_VALID_URL = r'https?://(?:www\.)?cartoonnetwork\.com/video/(?:[^/]+/)+(?P<id>[^/?#]+)-(?:clip|episode)\.html'
_TEST = {
'url': 'https://www.cartoonnetwork.com/video/ben-10/how-to-draw-upgrade-episode.html',
'info_dict': {
'id': '6e3375097f63874ebccec7ef677c1c3845fa850e',
'ext': 'mp4',
'title': 'How to Draw Upgrade',
'description': 'md5:2061d83776db7e8be4879684eefe8c0f',
},
'params': {
# m3u8 download
'skip_download': True,
},
}
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
def find_field(global_re, name, content_re=None, value_re='[^"]+', fatal=False):
metadata_re = ''
if content_re:
metadata_re = r'|video_metadata\.content_' + content_re
return self._search_regex(
rf'(?:_cnglobal\.currentVideo\.{global_re}{metadata_re})\s*=\s*"({value_re})";',
webpage, name, fatal=fatal)
media_id = find_field('mediaId', 'media id', 'id', '[0-9a-f]{40}', True)
title = find_field('episodeTitle', 'title', '(?:episodeName|name)', fatal=True)
info = self._extract_ngtv_info(
media_id, {'networkId': 'cartoonnetwork'}, {
'url': url,
'site_name': 'CartoonNetwork',
'auth_required': find_field('authType', 'auth type') != 'unauth',
})
series = find_field(
'propertyName', 'series', 'showName') or self._html_search_meta('partOfSeries', webpage)
info.update({
'id': media_id,
'display_id': display_id,
'title': title,
'description': self._html_search_meta('description', webpage),
'series': series,
'episode': title,
})
for field in ('season', 'episode'):
field_name = field + 'Number'
info[field + '_number'] = int_or_none(find_field(
field_name, field + ' number', value_re=r'\d+') or self._html_search_meta(field_name, webpage))
return info
plugins/youtube_download/yt_dlp/extractor/cbc.py
+61 -5
View File
@@ -31,7 +31,7 @@ from ..utils.traversal import require, traverse_obj, trim_str
class CBCIE(InfoExtractor): class CBCIE(InfoExtractor):
IE_NAME = 'cbc.ca' IE_NAME = 'cbc.ca'
_VALID_URL = r'https?://(?:www\.)?cbc\.ca/(?!player/)(?:[^/]+/)+(?P<id>[^/?#]+)' _VALID_URL = r'https?://(?:www\.)?cbc\.ca/(?!player/|listen/|i/caffeine/syndicate/)(?:[^/?#]+/)+(?P<id>[^/?#]+)'
_TESTS = [{ _TESTS = [{
# with mediaId # with mediaId
'url': 'http://www.cbc.ca/22minutes/videos/clips-season-23/don-cherry-play-offs', 'url': 'http://www.cbc.ca/22minutes/videos/clips-season-23/don-cherry-play-offs',
@@ -112,10 +112,6 @@ class CBCIE(InfoExtractor):
'playlist_mincount': 6, 'playlist_mincount': 6,
}] }]
@classmethod
def suitable(cls, url):
return False if CBCPlayerIE.suitable(url) else super().suitable(url)
def _extract_player_init(self, player_init, display_id): def _extract_player_init(self, player_init, display_id):
player_info = self._parse_json(player_init, display_id, js_to_json) player_info = self._parse_json(player_init, display_id, js_to_json)
media_id = player_info.get('mediaId') media_id = player_info.get('mediaId')
@@ -913,3 +909,63 @@ class CBCGemLiveIE(InfoExtractor):
'thumbnail': ('images', 'card', 'url'), 'thumbnail': ('images', 'card', 'url'),
}), }),
} }
class CBCListenIE(InfoExtractor):
IE_NAME = 'cbc.ca:listen'
_VALID_URL = r'https?://(?:www\.)?cbc\.ca/listen/(?:cbc-podcasts|live-radio)/[\w-]+/[\w-]+/(?P<id>\d+)'
_TESTS = [{
'url': 'https://www.cbc.ca/listen/cbc-podcasts/1353-the-naked-emperor/episode/16142603-introducing-understood-who-broke-the-internet',
'info_dict': {
'id': '16142603',
'title': 'Introducing Understood: Who Broke the Internet?',
'ext': 'mp3',
'description': 'md5:c605117500084e43f08a950adc6a708c',
'duration': 229,
'timestamp': 1745812800,
'release_timestamp': 1745827200,
'release_date': '20250428',
'upload_date': '20250428',
},
}, {
'url': 'https://www.cbc.ca/listen/live-radio/1-64-the-house/clip/16170773-should-canada-suck-stand-donald-trump',
'info_dict': {
'id': '16170773',
'title': 'Should Canada suck up or stand up to Donald Trump?',
'ext': 'mp3',
'description': 'md5:7385194f1cdda8df27ba3764b35e7976',
'duration': 3159,
'timestamp': 1758340800,
'release_timestamp': 1758254400,
'release_date': '20250919',
'upload_date': '20250920',
},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
response = self._download_json(
f'https://www.cbc.ca/listen/api/v1/clips/{video_id}', video_id, fatal=False)
data = traverse_obj(response, ('data', {dict}))
if not data:
self.report_warning('API failed to return data. Falling back to webpage parsing')
webpage = self._download_webpage(url, video_id)
preloaded_state = self._search_json(
r'window\.__PRELOADED_STATE__\s*=', webpage, 'preloaded state',
video_id, transform_source=js_to_json)
data = traverse_obj(preloaded_state, (
('podcastDetailData', 'showDetailData'), ..., 'episodes',
lambda _, v: str(v['clipID']) == video_id, any, {require('episode data')}))
return {
'id': video_id,
**traverse_obj(data, {
'url': (('src', 'url'), {url_or_none}, any),
'title': ('title', {str}),
'description': ('description', {str}),
'release_timestamp': ('releasedAt', {int_or_none(scale=1000)}),
'timestamp': ('airdate', {int_or_none(scale=1000)}),
'duration': ('duration', {int_or_none}),
}),
}
plugins/youtube_download/yt_dlp/extractor/cbsnews.py
+1 -9
View File
@@ -5,8 +5,6 @@ import zlib
from .anvato import AnvatoIE from .anvato import AnvatoIE
from .common import InfoExtractor from .common import InfoExtractor
from .paramountplus import ParamountPlusIE
from ..networking import HEADRequest
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
UserNotLive, UserNotLive,
@@ -132,13 +130,7 @@ class CBSNewsEmbedIE(CBSNewsBaseIE):
video_id = item['mpxRefId'] video_id = item['mpxRefId']
video_url = self._get_video_url(item) video_url = self._get_video_url(item)
if not video_url: if not video_url:
# Old embeds redirect user to ParamountPlus but most links are 404 raise ExtractorError('This video is no longer available', expected=True)
pplus_url = f'https://www.paramountplus.com/shows/video/{video_id}'
try:
self._request_webpage(HEADRequest(pplus_url), video_id)
return self.url_result(pplus_url, ParamountPlusIE)
except ExtractorError:
self.raise_no_formats('This video is no longer available', True, video_id)
return self._extract_video(item, video_url, video_id) return self._extract_video(item, video_url, video_id)
plugins/youtube_download/yt_dlp/extractor/cellebrite.py
-12
View File
@@ -5,18 +5,6 @@ from ..utils import ExtractorError, make_archive_id, url_basename
class CellebriteIE(VidyardBaseIE): class CellebriteIE(VidyardBaseIE):
_VALID_URL = r'https?://cellebrite\.com/(?:\w+)?/(?P<id>[\w-]+)' _VALID_URL = r'https?://cellebrite\.com/(?:\w+)?/(?P<id>[\w-]+)'
_TESTS = [{ _TESTS = [{
'url': 'https://cellebrite.com/en/collect-data-from-android-devices-with-cellebrite-ufed/',
'info_dict': {
'id': 'ZqmUss3dQfEMGpauambPuH',
'display_id': '16025876',
'ext': 'mp4',
'title': 'Ask the Expert: Chat Capture - Collect Data from Android Devices in Cellebrite UFED',
'description': 'md5:dee48fe12bbae5c01fe6a053f7676da4',
'thumbnail': 'https://cellebrite.com/wp-content/uploads/2021/05/Chat-Capture-1024x559.png',
'duration': 455.979,
'_old_archive_ids': ['cellebrite 16025876'],
},
}, {
'url': 'https://cellebrite.com/en/how-to-lawfully-collect-the-maximum-amount-of-data-from-android-devices/', 'url': 'https://cellebrite.com/en/how-to-lawfully-collect-the-maximum-amount-of-data-from-android-devices/',
'info_dict': { 'info_dict': {
'id': 'QV1U8a2yzcxigw7VFnqKyg', 'id': 'QV1U8a2yzcxigw7VFnqKyg',
plugins/youtube_download/yt_dlp/extractor/charlierose.py
+5 -1
View File
@@ -6,7 +6,7 @@ class CharlieRoseIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?charlierose\.com/(?:video|episode)(?:s|/player)/(?P<id>\d+)' _VALID_URL = r'https?://(?:www\.)?charlierose\.com/(?:video|episode)(?:s|/player)/(?P<id>\d+)'
_TESTS = [{ _TESTS = [{
'url': 'https://charlierose.com/videos/27996', 'url': 'https://charlierose.com/videos/27996',
'md5': 'fda41d49e67d4ce7c2411fd2c4702e09', 'md5': '4405b662f557f94aa256fa6a7baf7426',
'info_dict': { 'info_dict': {
'id': '27996', 'id': '27996',
'ext': 'mp4', 'ext': 'mp4',
@@ -39,12 +39,16 @@ class CharlieRoseIE(InfoExtractor):
self._PLAYER_BASE % video_id, webpage, video_id, self._PLAYER_BASE % video_id, webpage, video_id,
m3u8_entry_protocol='m3u8_native')[0] m3u8_entry_protocol='m3u8_native')[0]
self._remove_duplicate_formats(info_dict['formats']) self._remove_duplicate_formats(info_dict['formats'])
for fmt in info_dict['formats']:
if fmt.get('protocol') == 'm3u8_native':
fmt['__needs_testing'] = True
info_dict.update({ info_dict.update({
'id': video_id, 'id': video_id,
'title': title, 'title': title,
'thumbnail': self._og_search_thumbnail(webpage), 'thumbnail': self._og_search_thumbnail(webpage),
'description': self._og_search_description(webpage), 'description': self._og_search_description(webpage),
'_format_sort_fields': ('proto',),
}) })
return info_dict return info_dict
plugins/youtube_download/yt_dlp/extractor/cloudflarestream.py
+14 -6
View File
@@ -19,18 +19,16 @@ class CloudflareStreamIE(InfoExtractor):
'id': '31c9291ab41fac05471db4e73aa11717', 'id': '31c9291ab41fac05471db4e73aa11717',
'ext': 'mp4', 'ext': 'mp4',
'title': '31c9291ab41fac05471db4e73aa11717', 'title': '31c9291ab41fac05471db4e73aa11717',
'thumbnail': 'https://cloudflarestream.com/31c9291ab41fac05471db4e73aa11717/thumbnails/thumbnail.jpg', 'thumbnail': r're:https?://cloudflarestream\.com/.+\.jpg',
},
'params': {
'skip_download': 'm3u8',
}, },
'params': {'skip_download': 'm3u8'},
}, { }, {
'url': 'https://watch.cloudflarestream.com/embed/sdk-iframe-integration.fla9.latest.js?video=0e8e040aec776862e1d632a699edf59e', 'url': 'https://watch.cloudflarestream.com/embed/sdk-iframe-integration.fla9.latest.js?video=0e8e040aec776862e1d632a699edf59e',
'info_dict': { 'info_dict': {
'id': '0e8e040aec776862e1d632a699edf59e', 'id': '0e8e040aec776862e1d632a699edf59e',
'ext': 'mp4', 'ext': 'mp4',
'title': '0e8e040aec776862e1d632a699edf59e', 'title': '0e8e040aec776862e1d632a699edf59e',
'thumbnail': 'https://cloudflarestream.com/0e8e040aec776862e1d632a699edf59e/thumbnails/thumbnail.jpg', 'thumbnail': r're:https?://cloudflarestream\.com/.+\.jpg',
}, },
}, { }, {
'url': 'https://watch.cloudflarestream.com/9df17203414fd1db3e3ed74abbe936c1', 'url': 'https://watch.cloudflarestream.com/9df17203414fd1db3e3ed74abbe936c1',
@@ -54,11 +52,21 @@ class CloudflareStreamIE(InfoExtractor):
'id': 'eaef9dea5159cf968be84241b5cedfe7', 'id': 'eaef9dea5159cf968be84241b5cedfe7',
'ext': 'mp4', 'ext': 'mp4',
'title': 'eaef9dea5159cf968be84241b5cedfe7', 'title': 'eaef9dea5159cf968be84241b5cedfe7',
'thumbnail': 'https://cloudflarestream.com/eaef9dea5159cf968be84241b5cedfe7/thumbnails/thumbnail.jpg', 'thumbnail': r're:https?://cloudflarestream\.com/.+\.jpg',
}, },
'params': { 'params': {
'extractor_args': {'generic': {'impersonate': ['chrome']}},
'skip_download': 'm3u8', 'skip_download': 'm3u8',
}, },
}, {
# FIXME: Embed detection
'url': 'https://www.cloudflare.com/developer-platform/products/cloudflare-stream/',
'info_dict': {
'id': 'e7bd2dd67e0f8860b4ae81e33a966049',
'ext': 'mp4',
'title': 'e7bd2dd67e0f8860b4ae81e33a966049',
'thumbnail': r're:https?://cloudflarestream\.com/.+\.jpg',
},
}] }]
def _real_extract(self, url): def _real_extract(self, url):
plugins/youtube_download/yt_dlp/extractor/cloudycdn.py
+19 -7
View File
@@ -11,7 +11,7 @@ from ..utils.traversal import traverse_obj
class CloudyCDNIE(InfoExtractor): class CloudyCDNIE(InfoExtractor):
_VALID_URL = r'(?:https?:)?//embed\.cloudycdn\.services/(?P<site_id>[^/?#]+)/media/(?P<id>[\w-]+)' _VALID_URL = r'(?:https?:)?//embed\.(?P<domain>cloudycdn\.services|backscreen\.com)/(?P<site_id>[^/?#]+)/media/(?P<id>[\w-]+)'
_EMBED_REGEX = [rf'<iframe[^>]+\bsrc=[\'"](?P<url>{_VALID_URL})'] _EMBED_REGEX = [rf'<iframe[^>]+\bsrc=[\'"](?P<url>{_VALID_URL})']
_TESTS = [{ _TESTS = [{
'url': 'https://embed.cloudycdn.services/ltv/media/46k_d23-6000-105?', 'url': 'https://embed.cloudycdn.services/ltv/media/46k_d23-6000-105?',
@@ -23,7 +23,7 @@ class CloudyCDNIE(InfoExtractor):
'duration': 1442, 'duration': 1442,
'upload_date': '20231121', 'upload_date': '20231121',
'title': 'D23-6000-105_cetstud', 'title': 'D23-6000-105_cetstud',
'thumbnail': 'https://store.cloudycdn.services/tmsp00060/assets/media/660858/placeholder1700589200.jpg', 'thumbnail': 'https://store.bstrm.net/tmsp00060/assets/media/660858/placeholder1700589200.jpg',
}, },
}, { }, {
'url': 'https://embed.cloudycdn.services/izm/media/26e_lv-8-5-1', 'url': 'https://embed.cloudycdn.services/izm/media/26e_lv-8-5-1',
@@ -33,7 +33,7 @@ class CloudyCDNIE(InfoExtractor):
'ext': 'mp4', 'ext': 'mp4',
'title': 'LV-8-5-1', 'title': 'LV-8-5-1',
'timestamp': 1669767167, 'timestamp': 1669767167,
'thumbnail': 'https://store.cloudycdn.services/tmsp00120/assets/media/488306/placeholder1679423604.jpg', 'thumbnail': 'https://store.bstrm.net/tmsp00120/assets/media/488306/placeholder1679423604.jpg',
'duration': 1205, 'duration': 1205,
'upload_date': '20221130', 'upload_date': '20221130',
}, },
@@ -48,9 +48,21 @@ class CloudyCDNIE(InfoExtractor):
'duration': 1673, 'duration': 1673,
'title': 'D24-6000-074-cetstud', 'title': 'D24-6000-074-cetstud',
'timestamp': 1718902233, 'timestamp': 1718902233,
'thumbnail': 'https://store.cloudycdn.services/tmsp00060/assets/media/788392/placeholder1718903938.jpg', 'thumbnail': 'https://store.bstrm.net/tmsp00060/assets/media/788392/placeholder1718903938.jpg',
}, },
'params': {'format': 'bv'}, 'params': {'format': 'bv'},
}, {
'url': 'https://embed.backscreen.com/ltv/media/32j_z25-0600-127?',
'md5': '9b6fa09ac1a4de53d4f42b94affc3b42',
'info_dict': {
'id': '32j_z25-0600-127',
'ext': 'mp4',
'title': 'Z25-0600-127-DZ',
'duration': 1906,
'thumbnail': 'https://store.bstrm.net/tmsp00060/assets/media/977427/placeholder1746633646.jpg',
'timestamp': 1746632402,
'upload_date': '20250507',
},
}] }]
_WEBPAGE_TESTS = [{ _WEBPAGE_TESTS = [{
'url': 'https://www.tavaklase.lv/video/es-esmu-mina-um-2/', 'url': 'https://www.tavaklase.lv/video/es-esmu-mina-um-2/',
@@ -60,17 +72,17 @@ class CloudyCDNIE(InfoExtractor):
'ext': 'mp4', 'ext': 'mp4',
'upload_date': '20230223', 'upload_date': '20230223',
'duration': 629, 'duration': 629,
'thumbnail': 'https://store.cloudycdn.services/tmsp00120/assets/media/518407/placeholder1678748124.jpg', 'thumbnail': 'https://store.bstrm.net/tmsp00120/assets/media/518407/placeholder1678748124.jpg',
'timestamp': 1677181513, 'timestamp': 1677181513,
'title': 'LIB-2', 'title': 'LIB-2',
}, },
}] }]
def _real_extract(self, url): def _real_extract(self, url):
site_id, video_id = self._match_valid_url(url).group('site_id', 'id') domain, site_id, video_id = self._match_valid_url(url).group('domain', 'site_id', 'id')
data = self._download_json( data = self._download_json(
f'https://player.cloudycdn.services/player/{site_id}/media/{video_id}/', f'https://player.{domain}/player/{site_id}/media/{video_id}/',
video_id, data=urlencode_postdata({ video_id, data=urlencode_postdata({
'version': '6.4.0', 'version': '6.4.0',
'referer': url, 'referer': url,
plugins/youtube_download/yt_dlp/extractor/cmt.py
-55
View File
@@ -1,55 +0,0 @@
from .mtv import MTVIE
# TODO: Remove - Reason: Outdated Site
class CMTIE(MTVIE): # XXX: Do not subclass from concrete IE
_WORKING = False
IE_NAME = 'cmt.com'
_VALID_URL = r'https?://(?:www\.)?cmt\.com/(?:videos|shows|(?:full-)?episodes|video-clips)/(?P<id>[^/]+)'
_TESTS = [{
'url': 'http://www.cmt.com/videos/garth-brooks/989124/the-call-featuring-trisha-yearwood.jhtml#artist=30061',
'md5': 'e6b7ef3c4c45bbfae88061799bbba6c2',
'info_dict': {
'id': '989124',
'ext': 'mp4',
'title': 'Garth Brooks - "The Call (featuring Trisha Yearwood)"',
'description': 'Blame It All On My Roots',
},
'skip': 'Video not available',
}, {
'url': 'http://www.cmt.com/videos/misc/1504699/still-the-king-ep-109-in-3-minutes.jhtml#id=1739908',
'md5': 'e61a801ca4a183a466c08bd98dccbb1c',
'info_dict': {
'id': '1504699',
'ext': 'mp4',
'title': 'Still The King Ep. 109 in 3 Minutes',
'description': 'Relive or catch up with Still The King by watching this recap of season 1, episode 9.',
'timestamp': 1469421000.0,
'upload_date': '20160725',
},
}, {
'url': 'http://www.cmt.com/shows/party-down-south/party-down-south-ep-407-gone-girl/1738172/playlist/#id=1738172',
'only_matching': True,
}, {
'url': 'http://www.cmt.com/full-episodes/537qb3/nashville-the-wayfaring-stranger-season-5-ep-501',
'only_matching': True,
}, {
'url': 'http://www.cmt.com/video-clips/t9e4ci/nashville-juliette-in-2-minutes',
'only_matching': True,
}]
def _extract_mgid(self, webpage, url):
mgid = self._search_regex(
r'MTVN\.VIDEO\.contentUri\s*=\s*([\'"])(?P<mgid>.+?)\1',
webpage, 'mgid', group='mgid', default=None)
if not mgid:
mgid = self._extract_triforce_mgid(webpage)
return mgid
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
mgid = self._extract_mgid(webpage, url)
return self.url_result(f'http://media.mtvnservices.com/embed/{mgid}')
plugins/youtube_download/yt_dlp/extractor/cnn.py
+1
View File
@@ -272,6 +272,7 @@ class CNNIndonesiaIE(InfoExtractor):
return merge_dicts(json_ld_data, { return merge_dicts(json_ld_data, {
'_type': 'url_transparent', '_type': 'url_transparent',
'url': embed_url, 'url': embed_url,
'id': video_id,
'upload_date': upload_date, 'upload_date': upload_date,
'tags': try_call(lambda: self._html_search_meta('keywords', webpage).split(', ')), 'tags': try_call(lambda: self._html_search_meta('keywords', webpage).split(', ')),
}) })
plugins/youtube_download/yt_dlp/extractor/comedycentral.py
+20 -48
View File
@@ -1,55 +1,27 @@
from .mtv import MTVServicesInfoExtractor from .mtv import MTVServicesBaseIE
class ComedyCentralIE(MTVServicesInfoExtractor): class ComedyCentralIE(MTVServicesBaseIE):
_VALID_URL = r'https?://(?:www\.)?cc\.com/(?:episodes|video(?:-clips)?|collection-playlist|movies)/(?P<id>[0-9a-z]{6})' _VALID_URL = r'https?://(?:www\.)?cc\.com/video-clips/(?P<id>[\da-z]{6})'
_FEED_URL = 'http://comedycentral.com/feeds/mrss/'
_TESTS = [{ _TESTS = [{
'url': 'http://www.cc.com/video-clips/5ke9v2/the-daily-show-with-trevor-noah-doc-rivers-and-steve-ballmer---the-nba-player-strike', 'url': 'https://www.cc.com/video-clips/wl12cx',
'md5': 'b8acb347177c680ff18a292aa2166f80',
'info_dict': { 'info_dict': {
'id': '89ccc86e-1b02-4f83-b0c9-1d9592ecd025', 'id': 'dec6953e-80c8-43b3-96cd-05e9230e704d',
'ext': 'mp4', 'ext': 'mp4',
'title': 'The Daily Show with Trevor Noah|August 28, 2020|25|25149|Doc Rivers and Steve Ballmer - The NBA Player Strike', 'display_id': 'wl12cx',
'description': 'md5:5334307c433892b85f4f5e5ac9ef7498', 'title': 'Alison Brie and Dave Franco -"Together"- Extended Interview',
'timestamp': 1598670000, 'description': 'md5:ec68e38d3282f863de9cde0ce5cd231c',
'upload_date': '20200829', 'duration': 516.76,
'thumbnail': r're:https://images\.paramount\.tech/uri/mgid:arc:imageassetref:',
'series': 'The Daily Show',
'season': 'Season 30',
'season_number': 30,
'episode': 'Episode 0',
'episode_number': 0,
'timestamp': 1753973314,
'upload_date': '20250731',
'release_timestamp': 1753977914,
'release_date': '20250731',
}, },
}, { 'params': {'skip_download': 'm3u8'},
'url': 'http://www.cc.com/episodes/pnzzci/drawn-together--american-idol--parody-clip-show-season-3-ep-314',
'only_matching': True,
}, {
'url': 'https://www.cc.com/video/k3sdvm/the-daily-show-with-jon-stewart-exclusive-the-fourth-estate',
'only_matching': True,
}, {
'url': 'https://www.cc.com/collection-playlist/cosnej/stand-up-specials/t6vtjb',
'only_matching': True,
}, {
'url': 'https://www.cc.com/movies/tkp406/a-cluesterfuenke-christmas',
'only_matching': True,
}] }]
class ComedyCentralTVIE(MTVServicesInfoExtractor):
_VALID_URL = r'https?://(?:www\.)?comedycentral\.tv/folgen/(?P<id>[0-9a-z]{6})'
_TESTS = [{
'url': 'https://www.comedycentral.tv/folgen/pxdpec/josh-investigates-klimawandel-staffel-1-ep-1',
'info_dict': {
'id': '15907dc3-ec3c-11e8-a442-0e40cf2fc285',
'ext': 'mp4',
'title': 'Josh Investigates',
'description': 'Steht uns das Ende der Welt bevor?',
},
}]
_FEED_URL = 'http://feeds.mtvnservices.com/od/feed/intl-mrss-player-feed'
_GEO_COUNTRIES = ['DE']
def _get_feed_query(self, uri):
return {
'accountOverride': 'intl.mtvi.com',
'arcEp': 'web.cc.tv',
'ep': 'b9032c3a',
'imageEp': 'web.cc.tv',
'mgid': uri,
}
plugins/youtube_download/yt_dlp/extractor/common.py
+169 -43
View File
@@ -1,5 +1,6 @@
import base64 import base64
import collections import collections
import contextlib
import functools import functools
import getpass import getpass
import http.client import http.client
@@ -37,7 +38,6 @@ from ..networking.exceptions import (
TransportError, TransportError,
network_exceptions, network_exceptions,
) )
from ..networking.impersonate import ImpersonateTarget
from ..utils import ( from ..utils import (
IDENTITY, IDENTITY,
JSON_LD_RE, JSON_LD_RE,
@@ -101,6 +101,7 @@ from ..utils import (
xpath_with_ns, xpath_with_ns,
) )
from ..utils._utils import _request_dump_filename from ..utils._utils import _request_dump_filename
from ..utils.jslib import devalue
class InfoExtractor: class InfoExtractor:
@@ -242,7 +243,7 @@ class InfoExtractor:
* extra_param_to_segment_url A query string to append to each * extra_param_to_segment_url A query string to append to each
fragment's URL, or to update each existing query string fragment's URL, or to update each existing query string
with. If it is an HLS stream with an AES-128 decryption key, with. If it is an HLS stream with an AES-128 decryption key,
the query paramaters will be passed to the key URI as well, the query parameters will be passed to the key URI as well,
unless there is an `extra_param_to_key_url` given, unless there is an `extra_param_to_key_url` given,
or unless an external key URI is provided via `hls_aes`. or unless an external key URI is provided via `hls_aes`.
Only applied by the native HLS/DASH downloaders. Only applied by the native HLS/DASH downloaders.
@@ -257,11 +258,20 @@ class InfoExtractor:
* key The key (as hex) used to decrypt fragments. * key The key (as hex) used to decrypt fragments.
If `key` is given, any key URI will be ignored If `key` is given, any key URI will be ignored
* iv The IV (as hex) used to decrypt fragments * iv The IV (as hex) used to decrypt fragments
* impersonate Impersonate target(s). Can be any of the following entities:
* an instance of yt_dlp.networking.impersonate.ImpersonateTarget
* a string in the format of CLIENT[:OS]
* a list or a tuple of CLIENT[:OS] strings or ImpersonateTarget instances
* a boolean value; True means any impersonate target is sufficient
* available_at Unix timestamp of when a format will be available to download
* downloader_options A dictionary of downloader options * downloader_options A dictionary of downloader options
(For internal use only) (For internal use only)
* http_chunk_size Chunk size for HTTP downloads * http_chunk_size Chunk size for HTTP downloads
* ffmpeg_args Extra arguments for ffmpeg downloader (input) * ffmpeg_args Extra arguments for ffmpeg downloader (input)
* ffmpeg_args_out Extra arguments for ffmpeg downloader (output) * ffmpeg_args_out Extra arguments for ffmpeg downloader (output)
* ws (NiconicoLiveFD only) WebSocketResponse
* ws_url (NiconicoLiveFD only) Websockets URL
* max_quality (NiconicoLiveFD only) Max stream quality string
* is_dash_periods Whether the format is a result of merging * is_dash_periods Whether the format is a result of merging
multiple DASH periods. multiple DASH periods.
RTMP formats can also have the additional fields: page_url, RTMP formats can also have the additional fields: page_url,
@@ -331,6 +341,7 @@ class InfoExtractor:
* "name": Name or description of the subtitles * "name": Name or description of the subtitles
* "http_headers": A dictionary of additional HTTP headers * "http_headers": A dictionary of additional HTTP headers
to add to the request. to add to the request.
* "impersonate": Impersonate target(s); same as the "formats" field
"ext" will be calculated from URL if missing "ext" will be calculated from URL if missing
automatic_captions: Like 'subtitles'; contains automatically generated automatic_captions: Like 'subtitles'; contains automatically generated
captions instead of normal subtitles captions instead of normal subtitles
@@ -387,6 +398,8 @@ class InfoExtractor:
chapters: A list of dictionaries, with the following entries: chapters: A list of dictionaries, with the following entries:
* "start_time" - The start time of the chapter in seconds * "start_time" - The start time of the chapter in seconds
* "end_time" - The end time of the chapter in seconds * "end_time" - The end time of the chapter in seconds
(optional: core code can determine this value from
the next chapter's start_time or the video's duration)
* "title" (optional, string) * "title" (optional, string)
heatmap: A list of dictionaries, with the following entries: heatmap: A list of dictionaries, with the following entries:
* "start_time" - The start time of the data point in seconds * "start_time" - The start time of the data point in seconds
@@ -401,12 +414,13 @@ class InfoExtractor:
'unlisted' or 'public'. Use 'InfoExtractor._availability' 'unlisted' or 'public'. Use 'InfoExtractor._availability'
to set it to set it
media_type: The type of media as classified by the site, e.g. "episode", "clip", "trailer" media_type: The type of media as classified by the site, e.g. "episode", "clip", "trailer"
_old_archive_ids: A list of old archive ids needed for backward compatibility _old_archive_ids: A list of old archive ids needed for backward
compatibility. Use yt_dlp.utils.make_archive_id to generate ids
_format_sort_fields: A list of fields to use for sorting formats _format_sort_fields: A list of fields to use for sorting formats
__post_extractor: A function to be called just before the metadata is __post_extractor: A function to be called just before the metadata is
written to either disk, logger or console. The function written to either disk, logger or console. The function
must return a dict which will be added to the info_dict. must return a dict which will be added to the info_dict.
This is usefull for additional information that is This is useful for additional information that is
time-consuming to extract. Note that the fields thus time-consuming to extract. Note that the fields thus
extracted will not be available to output template and extracted will not be available to output template and
match_filter. So, only "comments" and "comment_count" are match_filter. So, only "comments" and "comment_count" are
@@ -879,26 +893,17 @@ class InfoExtractor:
extensions = {} extensions = {}
if impersonate in (True, ''): available_target, requested_targets = self._downloader._parse_impersonate_targets(impersonate)
impersonate = ImpersonateTarget()
requested_targets = [
t if isinstance(t, ImpersonateTarget) else ImpersonateTarget.from_str(t)
for t in variadic(impersonate)
] if impersonate else []
available_target = next(filter(self._downloader._impersonate_target_available, requested_targets), None)
if available_target: if available_target:
extensions['impersonate'] = available_target extensions['impersonate'] = available_target
elif requested_targets: elif requested_targets:
message = 'The extractor is attempting impersonation, but ' msg = 'The extractor is attempting impersonation'
message += (
'no impersonate target is available' if not str(impersonate)
else f'none of these impersonate targets are available: "{", ".join(map(str, requested_targets))}"')
info_msg = ('see https://github.com/yt-dlp/yt-dlp#impersonation '
'for information on installing the required dependencies')
if require_impersonation: if require_impersonation:
raise ExtractorError(f'{message}; {info_msg}', expected=True) raise ExtractorError(
self.report_warning(f'{message}; if you encounter errors, then {info_msg}', only_once=True) self._downloader._unavailable_targets_message(requested_targets, note=msg, is_error=True),
expected=True)
self.report_warning(
self._downloader._unavailable_targets_message(requested_targets, note=msg), only_once=True)
try: try:
return self._downloader.urlopen(self._create_request(url_or_request, data, headers, query, extensions)) return self._downloader.urlopen(self._create_request(url_or_request, data, headers, query, extensions))
@@ -1523,11 +1528,11 @@ class InfoExtractor:
r'>\s*(?:18\s+U(?:\.S\.C\.|SC)\s+)?(?:§+\s*)?2257\b', r'>\s*(?:18\s+U(?:\.S\.C\.|SC)\s+)?(?:§+\s*)?2257\b',
] ]
age_limit = 0 age_limit = None
for marker in AGE_LIMIT_MARKERS: for marker in AGE_LIMIT_MARKERS:
mobj = re.search(marker, html) mobj = re.search(marker, html)
if mobj: if mobj:
age_limit = max(age_limit, int(traverse_obj(mobj, 1, default=18))) age_limit = max(age_limit or 0, int(traverse_obj(mobj, 1, default=18)))
return age_limit return age_limit
def _media_rating_search(self, html): def _media_rating_search(self, html):
@@ -1658,7 +1663,7 @@ class InfoExtractor:
'end_time': part.get('endOffset'), 'end_time': part.get('endOffset'),
} for part in variadic(e.get('hasPart') or []) if part.get('@type') == 'Clip'] } for part in variadic(e.get('hasPart') or []) if part.get('@type') == 'Clip']
for idx, (last_c, current_c, next_c) in enumerate(zip( for idx, (last_c, current_c, next_c) in enumerate(zip(
[{'end_time': 0}, *chapters], chapters, chapters[1:])): [{'end_time': 0}, *chapters], chapters, chapters[1:], strict=False)):
current_c['end_time'] = current_c['end_time'] or next_c['start_time'] current_c['end_time'] = current_c['end_time'] or next_c['start_time']
current_c['start_time'] = current_c['start_time'] or last_c['end_time'] current_c['start_time'] = current_c['start_time'] or last_c['end_time']
if None in current_c.values(): if None in current_c.values():
@@ -1675,9 +1680,9 @@ class InfoExtractor:
'ext': mimetype2ext(e.get('encodingFormat')), 'ext': mimetype2ext(e.get('encodingFormat')),
'title': unescapeHTML(e.get('name')), 'title': unescapeHTML(e.get('name')),
'description': unescapeHTML(e.get('description')), 'description': unescapeHTML(e.get('description')),
'thumbnails': [{'url': unescapeHTML(url)} 'thumbnails': traverse_obj(e, (('thumbnailUrl', 'thumbnailURL', 'thumbnail_url'), (None, ...), {
for url in variadic(traverse_obj(e, 'thumbnailUrl', 'thumbnailURL')) 'url': ({str}, {unescapeHTML}, {self._proto_relative_url}, {url_or_none}),
if url_or_none(url)], })),
'duration': parse_duration(e.get('duration')), 'duration': parse_duration(e.get('duration')),
'timestamp': unified_timestamp(e.get('uploadDate')), 'timestamp': unified_timestamp(e.get('uploadDate')),
# author can be an instance of 'Organization' or 'Person' types. # author can be an instance of 'Organization' or 'Person' types.
@@ -1778,6 +1783,59 @@ class InfoExtractor:
r'<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>', webpage, 'next.js data', r'<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>', webpage, 'next.js data',
video_id, end_pattern='</script>', fatal=fatal, default=default, **kw) video_id, end_pattern='</script>', fatal=fatal, default=default, **kw)
def _search_nextjs_v13_data(self, webpage, video_id, fatal=True):
"""Parses Next.js app router flight data that was introduced in Next.js v13"""
nextjs_data = {}
if not fatal and not isinstance(webpage, str):
return nextjs_data
def flatten(flight_data):
if not isinstance(flight_data, list):
return
if len(flight_data) == 4 and flight_data[0] == '$':
_, name, _, data = flight_data
if not isinstance(data, dict):
return
children = data.pop('children', None)
if data and isinstance(name, str) and re.fullmatch(r'\$L[0-9a-f]+', name):
# It is useful hydration JSON data
nextjs_data[name[2:]] = data
flatten(children)
return
for f in flight_data:
flatten(f)
flight_text = ''
# The pattern for the surrounding JS/tag should be strict as it's a hardcoded string in the next.js source
# Ref: https://github.com/vercel/next.js/blob/5a4a08fdc/packages/next/src/server/app-render/use-flight-response.tsx#L189
for flight_segment in re.findall(r'<script\b[^>]*>self\.__next_f\.push\((\[.+?\])\)</script>', webpage):
segment = self._parse_json(flight_segment, video_id, fatal=fatal, errnote=None if fatal else False)
# Some earlier versions of next.js "optimized" away this array structure; this is unsupported
# Ref: https://github.com/vercel/next.js/commit/0123a9d5c9a9a77a86f135b7ae30b46ca986d761
if not isinstance(segment, list) or len(segment) != 2:
self.write_debug(
f'{video_id}: Unsupported next.js flight data structure detected', only_once=True)
continue
# Only use the relevant payload type (1 == data)
# Ref: https://github.com/vercel/next.js/blob/5a4a08fdc/packages/next/src/server/app-render/use-flight-response.tsx#L11-L14
payload_type, chunk = segment
if payload_type == 1:
flight_text += chunk
for f in flight_text.splitlines():
prefix, _, body = f.lstrip().partition(':')
if not re.fullmatch(r'[0-9a-f]+', prefix):
continue
# The body still isn't guaranteed to be valid JSON, so parsing should always be non-fatal
if body.startswith('[') and body.endswith(']'):
flatten(self._parse_json(body, video_id, fatal=False, errnote=False))
elif body.startswith('{') and body.endswith('}'):
data = self._parse_json(body, video_id, fatal=False, errnote=False)
if data is not None:
nextjs_data[prefix] = data
return nextjs_data
def _search_nuxt_data(self, webpage, video_id, context_name='__NUXT__', *, fatal=True, traverse=('data', 0)): def _search_nuxt_data(self, webpage, video_id, context_name='__NUXT__', *, fatal=True, traverse=('data', 0)):
"""Parses Nuxt.js metadata. This works as long as the function __NUXT__ invokes is a pure function""" """Parses Nuxt.js metadata. This works as long as the function __NUXT__ invokes is a pure function"""
rectx = re.escape(context_name) rectx = re.escape(context_name)
@@ -1790,11 +1848,68 @@ class InfoExtractor:
return {} return {}
args = dict(zip(arg_keys.split(','), map(json.dumps, self._parse_json( args = dict(zip(arg_keys.split(','), map(json.dumps, self._parse_json(
f'[{arg_vals}]', video_id, transform_source=js_to_json, fatal=fatal) or ()))) f'[{arg_vals}]', video_id, transform_source=js_to_json, fatal=fatal) or ()), strict=True))
ret = self._parse_json(js, video_id, transform_source=functools.partial(js_to_json, vars=args), fatal=fatal) ret = self._parse_json(js, video_id, transform_source=functools.partial(js_to_json, vars=args), fatal=fatal)
return traverse_obj(ret, traverse) or {} return traverse_obj(ret, traverse) or {}
def _resolve_nuxt_array(self, array, video_id, *, fatal=True, default=NO_DEFAULT):
"""Resolves Nuxt rich JSON payload arrays"""
# Ref: https://github.com/nuxt/nuxt/commit/9e503be0f2a24f4df72a3ccab2db4d3e63511f57
# https://github.com/nuxt/nuxt/pull/19205
if default is not NO_DEFAULT:
fatal = False
if not isinstance(array, list) or not array:
error_msg = 'Unable to resolve Nuxt JSON data: invalid input'
if fatal:
raise ExtractorError(error_msg, video_id=video_id)
elif default is NO_DEFAULT:
self.report_warning(error_msg, video_id=video_id)
return {} if default is NO_DEFAULT else default
def indirect_reviver(data):
return data
def json_reviver(data):
return json.loads(data)
gen = devalue.parse_iter(array, revivers={
'NuxtError': indirect_reviver,
'EmptyShallowRef': json_reviver,
'EmptyRef': json_reviver,
'ShallowRef': indirect_reviver,
'ShallowReactive': indirect_reviver,
'Ref': indirect_reviver,
'Reactive': indirect_reviver,
})
while True:
try:
error_msg = f'Error resolving Nuxt JSON: {gen.send(None)}'
if fatal:
raise ExtractorError(error_msg, video_id=video_id)
elif default is NO_DEFAULT:
self.report_warning(error_msg, video_id=video_id, only_once=True)
else:
self.write_debug(f'{video_id}: {error_msg}', only_once=True)
except StopIteration as error:
return error.value or ({} if default is NO_DEFAULT else default)
def _search_nuxt_json(self, webpage, video_id, *, fatal=True, default=NO_DEFAULT):
"""Parses metadata from Nuxt rich JSON payloads embedded in HTML"""
passed_default = default is not NO_DEFAULT
array = self._search_json(
r'<script\b[^>]+\bid="__NUXT_DATA__"[^>]*>', webpage,
'Nuxt JSON data', video_id, contains_pattern=r'\[(?s:.+)\]',
fatal=fatal, default=NO_DEFAULT if not passed_default else None)
if not array:
return default if passed_default else {}
return self._resolve_nuxt_array(array, video_id, fatal=fatal, default=default)
@staticmethod @staticmethod
def _hidden_inputs(html): def _hidden_inputs(html):
html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html) html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
@@ -2068,21 +2183,33 @@ class InfoExtractor:
raise ExtractorError(errnote, video_id=video_id) raise ExtractorError(errnote, video_id=video_id)
self.report_warning(f'{errnote}{bug_reports_message()}') self.report_warning(f'{errnote}{bug_reports_message()}')
return [], {} return [], {}
if note is None:
res = self._download_webpage_handle( note = 'Downloading m3u8 information'
m3u8_url, video_id, if errnote is None:
note='Downloading m3u8 information' if note is None else note, errnote = 'Failed to download m3u8 information'
errnote='Failed to download m3u8 information' if errnote is None else errnote, response = self._request_webpage(
m3u8_url, video_id, note=note, errnote=errnote,
fatal=fatal, data=data, headers=headers, query=query) fatal=fatal, data=data, headers=headers, query=query)
if response is False:
if res is False:
return [], {} return [], {}
m3u8_doc, urlh = res with contextlib.closing(response):
m3u8_url = urlh.url prefix = response.read(512)
if not prefix.startswith(b'#EXTM3U'):
msg = 'Response data has no m3u header'
if fatal:
raise ExtractorError(msg, video_id=video_id)
self.report_warning(f'{msg}{bug_reports_message()}', video_id=video_id)
return [], {}
content = self._webpage_read_content(
response, m3u8_url, video_id, note=note, errnote=errnote,
fatal=fatal, prefix=prefix, data=data)
if content is False:
return [], {}
return self._parse_m3u8_formats_and_subtitles( return self._parse_m3u8_formats_and_subtitles(
m3u8_doc, m3u8_url, ext=ext, entry_protocol=entry_protocol, content, response.url, ext=ext, entry_protocol=entry_protocol,
preference=preference, quality=quality, m3u8_id=m3u8_id, preference=preference, quality=quality, m3u8_id=m3u8_id,
note=note, errnote=errnote, fatal=fatal, live=live, data=data, note=note, errnote=errnote, fatal=fatal, live=live, data=data,
headers=headers, query=query, video_id=video_id) headers=headers, query=query, video_id=video_id)
@@ -2842,7 +2969,7 @@ class InfoExtractor:
else: else:
codecs = parse_codecs(codec_str) codecs = parse_codecs(codec_str)
if content_type not in ('video', 'audio', 'text'): if content_type not in ('video', 'audio', 'text'):
if mime_type == 'image/jpeg': if mime_type in ('image/avif', 'image/jpeg'):
content_type = mime_type content_type = mime_type
elif codecs.get('vcodec', 'none') != 'none': elif codecs.get('vcodec', 'none') != 'none':
content_type = 'video' content_type = 'video'
@@ -2902,14 +3029,14 @@ class InfoExtractor:
'manifest_url': mpd_url, 'manifest_url': mpd_url,
'filesize': filesize, 'filesize': filesize,
} }
elif content_type == 'image/jpeg': elif content_type in ('image/avif', 'image/jpeg'):
# See test case in VikiIE # See test case in VikiIE
# https://www.viki.com/videos/1175236v-choosing-spouse-by-lottery-episode-1 # https://www.viki.com/videos/1175236v-choosing-spouse-by-lottery-episode-1
f = { f = {
'format_id': format_id, 'format_id': format_id,
'ext': 'mhtml', 'ext': 'mhtml',
'manifest_url': mpd_url, 'manifest_url': mpd_url,
'format_note': 'DASH storyboards (jpeg)', 'format_note': f'DASH storyboards ({mimetype2ext(mime_type)})',
'acodec': 'none', 'acodec': 'none',
'vcodec': 'none', 'vcodec': 'none',
} }
@@ -2981,7 +3108,6 @@ class InfoExtractor:
else: else:
# $Number*$ or $Time$ in media template with S list available # $Number*$ or $Time$ in media template with S list available
# Example $Number*$: http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg # Example $Number*$: http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg
# Example $Time$: https://play.arkena.com/embed/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411
representation_ms_info['fragments'] = [] representation_ms_info['fragments'] = []
segment_time = 0 segment_time = 0
segment_d = None segment_d = None
@@ -3051,7 +3177,7 @@ class InfoExtractor:
'url': mpd_url or base_url, 'url': mpd_url or base_url,
'fragment_base_url': base_url, 'fragment_base_url': base_url,
'fragments': [], 'fragments': [],
'protocol': 'http_dash_segments' if mime_type != 'image/jpeg' else 'mhtml', 'protocol': 'mhtml' if mime_type in ('image/avif', 'image/jpeg') else 'http_dash_segments',
}) })
if 'initialization_url' in representation_ms_info: if 'initialization_url' in representation_ms_info:
initialization_url = representation_ms_info['initialization_url'] initialization_url = representation_ms_info['initialization_url']
@@ -3066,7 +3192,7 @@ class InfoExtractor:
else: else:
# Assuming direct URL to unfragmented media. # Assuming direct URL to unfragmented media.
f['url'] = base_url f['url'] = base_url
if content_type in ('video', 'audio', 'image/jpeg'): if content_type in ('video', 'audio', 'image/avif', 'image/jpeg'):
f['manifest_stream_number'] = stream_numbers[f['url']] f['manifest_stream_number'] = stream_numbers[f['url']]
stream_numbers[f['url']] += 1 stream_numbers[f['url']] += 1
period_entry['formats'].append(f) period_entry['formats'].append(f)
plugins/youtube_download/yt_dlp/extractor/condenast.py
+18
View File
@@ -96,6 +96,24 @@ class CondeNastIE(InfoExtractor):
'upload_date': '20150916', 'upload_date': '20150916',
'timestamp': 1442434920, 'timestamp': 1442434920,
}, },
}, {
# FIXME: Subtitles
'url': 'https://www.vanityfair.com/video/watch/vf-quiz-show-squid-game-s3',
'info_dict': {
'id': '6862f999c1afbc5ff06b4803',
'ext': 'mp4',
'title': '\'Squid Game\' Cast Tests How Well They Know Each Other',
'categories': ['Arts & Culture', 'Hollywood'],
'description': 'md5:7a9c668a1fc87648e77da13842ec1534',
'duration': 955,
'season': 'Season 1',
'series': 'Quizzing Each Other',
'tags': 'count:2',
'thumbnail': r're:https?://dwgyu36up6iuz\.cloudfront\.net/.+\.jpg',
'timestamp': 1751341306,
'upload_date': '20250701',
'uploader': 'vanityfair',
},
}, { }, {
'url': 'https://player.cnevids.com/inline/video/59138decb57ac36b83000005.js?target=js-cne-player', 'url': 'https://player.cnevids.com/inline/video/59138decb57ac36b83000005.js?target=js-cne-player',
'only_matching': True, 'only_matching': True,
plugins/youtube_download/yt_dlp/extractor/crackle.py
-243
View File
@@ -1,243 +0,0 @@
import hashlib
import hmac
import re
import time
from .common import InfoExtractor
from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
determine_ext,
float_or_none,
int_or_none,
orderedSet,
parse_age_limit,
parse_duration,
url_or_none,
)
class CrackleIE(InfoExtractor):
_VALID_URL = r'(?:crackle:|https?://(?:(?:www|m)\.)?(?:sony)?crackle\.com/(?:playlist/\d+/|(?:[^/]+/)+))(?P<id>\d+)'
_TESTS = [{
# Crackle is available in the United States and territories
'url': 'https://www.crackle.com/thanksgiving/2510064',
'info_dict': {
'id': '2510064',
'ext': 'mp4',
'title': 'Touch Football',
'description': 'md5:cfbb513cf5de41e8b56d7ab756cff4df',
'duration': 1398,
'view_count': int,
'average_rating': 0,
'age_limit': 17,
'genre': 'Comedy',
'creator': 'Daniel Powell',
'artist': 'Chris Elliott, Amy Sedaris',
'release_year': 2016,
'series': 'Thanksgiving',
'episode': 'Touch Football',
'season_number': 1,
'episode_number': 1,
},
'params': {
# m3u8 download
'skip_download': True,
},
'expected_warnings': [
'Trying with a list of known countries',
],
}, {
'url': 'https://www.sonycrackle.com/thanksgiving/2510064',
'only_matching': True,
}]
_MEDIA_FILE_SLOTS = {
'360p.mp4': {
'width': 640,
'height': 360,
},
'480p.mp4': {
'width': 768,
'height': 432,
},
'480p_1mbps.mp4': {
'width': 852,
'height': 480,
},
}
def _download_json(self, url, *args, **kwargs):
# Authorization generation algorithm is reverse engineered from:
# https://www.sonycrackle.com/static/js/main.ea93451f.chunk.js
timestamp = time.strftime('%Y%m%d%H%M', time.gmtime())
h = hmac.new(b'IGSLUQCBDFHEOIFM', '|'.join([url, timestamp]).encode(), hashlib.sha1).hexdigest().upper()
headers = {
'Accept': 'application/json',
'Authorization': '|'.join([h, timestamp, '117', '1']),
}
return InfoExtractor._download_json(self, url, *args, headers=headers, **kwargs)
def _real_extract(self, url):
video_id = self._match_id(url)
geo_bypass_country = self.get_param('geo_bypass_country', None)
countries = orderedSet((geo_bypass_country, 'US', 'AU', 'CA', 'AS', 'FM', 'GU', 'MP', 'PR', 'PW', 'MH', 'VI', ''))
num_countries, num = len(countries) - 1, 0
media = {}
for num, country in enumerate(countries):
if num == 1: # start hard-coded list
self.report_warning('%s. Trying with a list of known countries' % (
f'Unable to obtain video formats from {geo_bypass_country} API' if geo_bypass_country
else 'No country code was given using --geo-bypass-country'))
elif num == num_countries: # end of list
geo_info = self._download_json(
'https://web-api-us.crackle.com/Service.svc/geo/country',
video_id, fatal=False, note='Downloading geo-location information from crackle API',
errnote='Unable to fetch geo-location information from crackle') or {}
country = geo_info.get('CountryCode')
if country is None:
continue
self.to_screen(f'{self.IE_NAME} identified country as {country}')
if country in countries:
self.to_screen(f'Downloading from {country} API was already attempted. Skipping...')
continue
if country is None:
continue
try:
media = self._download_json(
f'https://web-api-us.crackle.com/Service.svc/details/media/{video_id}/{country}?disableProtocols=true',
video_id, note=f'Downloading media JSON from {country} API',
errnote='Unable to download media JSON')
except ExtractorError as e:
# 401 means geo restriction, trying next country
if isinstance(e.cause, HTTPError) and e.cause.status == 401:
continue
raise
status = media.get('status')
if status.get('messageCode') != '0':
raise ExtractorError(
'{} said: {} {} - {}'.format(
self.IE_NAME, status.get('messageCodeDescription'), status.get('messageCode'), status.get('message')),
expected=True)
# Found video formats
if isinstance(media.get('MediaURLs'), list):
break
ignore_no_formats = self.get_param('ignore_no_formats_error')
if not media or (not media.get('MediaURLs') and not ignore_no_formats):
raise ExtractorError(
'Unable to access the crackle API. Try passing your country code '
'to --geo-bypass-country. If it still does not work and the '
'video is available in your country')
title = media['Title']
formats, subtitles = [], {}
has_drm = False
for e in media.get('MediaURLs') or []:
if e.get('UseDRM'):
has_drm = True
format_url = url_or_none(e.get('DRMPath'))
else:
format_url = url_or_none(e.get('Path'))
if not format_url:
continue
ext = determine_ext(format_url)
if ext == 'm3u8':
fmts, subs = self._extract_m3u8_formats_and_subtitles(
format_url, video_id, 'mp4', entry_protocol='m3u8_native',
m3u8_id='hls', fatal=False)
formats.extend(fmts)
subtitles = self._merge_subtitles(subtitles, subs)
elif ext == 'mpd':
fmts, subs = self._extract_mpd_formats_and_subtitles(
format_url, video_id, mpd_id='dash', fatal=False)
formats.extend(fmts)
subtitles = self._merge_subtitles(subtitles, subs)
elif format_url.endswith('.ism/Manifest'):
fmts, subs = self._extract_ism_formats_and_subtitles(
format_url, video_id, ism_id='mss', fatal=False)
formats.extend(fmts)
subtitles = self._merge_subtitles(subtitles, subs)
else:
mfs_path = e.get('Type')
mfs_info = self._MEDIA_FILE_SLOTS.get(mfs_path)
if not mfs_info:
continue
formats.append({
'url': format_url,
'format_id': 'http-' + mfs_path.split('.')[0],
'width': mfs_info['width'],
'height': mfs_info['height'],
})
if not formats and has_drm:
self.report_drm(video_id)
description = media.get('Description')
duration = int_or_none(media.get(
'DurationInSeconds')) or parse_duration(media.get('Duration'))
view_count = int_or_none(media.get('CountViews'))
average_rating = float_or_none(media.get('UserRating'))
age_limit = parse_age_limit(media.get('Rating'))
genre = media.get('Genre')
release_year = int_or_none(media.get('ReleaseYear'))
creator = media.get('Directors')
artist = media.get('Cast')
if media.get('MediaTypeDisplayValue') == 'Full Episode':
series = media.get('ShowName')
episode = title
season_number = int_or_none(media.get('Season'))
episode_number = int_or_none(media.get('Episode'))
else:
series = episode = season_number = episode_number = None
cc_files = media.get('ClosedCaptionFiles')
if isinstance(cc_files, list):
for cc_file in cc_files:
if not isinstance(cc_file, dict):
continue
cc_url = url_or_none(cc_file.get('Path'))
if not cc_url:
continue
lang = cc_file.get('Locale') or 'en'
subtitles.setdefault(lang, []).append({'url': cc_url})
thumbnails = []
images = media.get('Images')
if isinstance(images, list):
for image_key, image_url in images.items():
mobj = re.search(r'Img_(\d+)[xX](\d+)', image_key)
if not mobj:
continue
thumbnails.append({
'url': image_url,
'width': int(mobj.group(1)),
'height': int(mobj.group(2)),
})
return {
'id': video_id,
'title': title,
'description': description,
'duration': duration,
'view_count': view_count,
'average_rating': average_rating,
'age_limit': age_limit,
'genre': genre,
'creator': creator,
'artist': artist,
'release_year': release_year,
'series': series,
'episode': episode,
'season_number': season_number,
'episode_number': episode_number,
'thumbnails': thumbnails,
'subtitles': subtitles,
'formats': formats,
}
plugins/youtube_download/yt_dlp/extractor/crooksandliars.py
+15 -2
View File
@@ -8,7 +8,6 @@ from ..utils import (
class CrooksAndLiarsIE(InfoExtractor): class CrooksAndLiarsIE(InfoExtractor):
_VALID_URL = r'https?://embed\.crooksandliars\.com/(?:embed|v)/(?P<id>[A-Za-z0-9]+)' _VALID_URL = r'https?://embed\.crooksandliars\.com/(?:embed|v)/(?P<id>[A-Za-z0-9]+)'
_EMBED_REGEX = [r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1'] _EMBED_REGEX = [r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1']
_TESTS = [{ _TESTS = [{
'url': 'https://embed.crooksandliars.com/embed/8RUoRhRi', 'url': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
'info_dict': { 'info_dict': {
@@ -16,7 +15,7 @@ class CrooksAndLiarsIE(InfoExtractor):
'ext': 'mp4', 'ext': 'mp4',
'title': 'Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!', 'title': 'Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!',
'description': 'md5:e1a46ad1650e3a5ec7196d432799127f', 'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
'thumbnail': r're:^https?://.*\.jpg', 'thumbnail': r're:https?://crooksandliars\.com/files/.+',
'timestamp': 1428207000, 'timestamp': 1428207000,
'upload_date': '20150405', 'upload_date': '20150405',
'uploader': 'Heather', 'uploader': 'Heather',
@@ -26,6 +25,20 @@ class CrooksAndLiarsIE(InfoExtractor):
'url': 'http://embed.crooksandliars.com/v/MTE3MjUtMzQ2MzA', 'url': 'http://embed.crooksandliars.com/v/MTE3MjUtMzQ2MzA',
'only_matching': True, 'only_matching': True,
}] }]
_WEBPAGE_TESTS = [{
'url': 'https://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
'info_dict': {
'id': '8RUoRhRi',
'ext': 'mp4',
'title': 'Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!',
'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
'duration': 236,
'thumbnail': r're:https?://crooksandliars\.com/files/.+',
'timestamp': 1428207000,
'upload_date': '20150405',
'uploader': 'Heather',
},
}]
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
plugins/youtube_download/yt_dlp/extractor/ctv.py
-49
View File
@@ -1,49 +0,0 @@
from .common import InfoExtractor
class CTVIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?ctv\.ca/(?P<id>(?:show|movie)s/[^/]+/[^/?#&]+)'
_TESTS = [{
'url': 'https://www.ctv.ca/shows/your-morning/wednesday-december-23-2020-s5e88',
'info_dict': {
'id': '2102249',
'ext': 'flv',
'title': 'Wednesday, December 23, 2020',
'thumbnail': r're:^https?://.*\.jpg$',
'description': 'Your Morning delivers original perspectives and unique insights into the headlines of the day.',
'timestamp': 1608732000,
'upload_date': '20201223',
'series': 'Your Morning',
'season': '2020-2021',
'season_number': 5,
'episode_number': 88,
'tags': ['Your Morning'],
'categories': ['Talk Show'],
'duration': 7467.126,
},
}, {
'url': 'https://www.ctv.ca/movies/adam-sandlers-eight-crazy-nights/adam-sandlers-eight-crazy-nights',
'only_matching': True,
}]
def _real_extract(self, url):
display_id = self._match_id(url)
content = self._download_json(
'https://www.ctv.ca/space-graphql/graphql', display_id, query={
'query': '''{
resolvedPath(path: "/%s") {
lastSegment {
content {
... on AxisContent {
axisId
videoPlayerDestCode
}
}
}
}
}''' % display_id, # noqa: UP031
})['data']['resolvedPath']['lastSegment']['content']
video_id = content['axisId']
return self.url_result(
'9c9media:{}:{}'.format(content['videoPlayerDestCode'], video_id),
'NineCNineMedia', video_id)
plugins/youtube_download/yt_dlp/extractor/cwtv.py
-180
View File
@@ -1,180 +0,0 @@
import re
from .common import InfoExtractor
from ..utils import (
ExtractorError,
int_or_none,
parse_age_limit,
parse_iso8601,
parse_qs,
smuggle_url,
str_or_none,
update_url_query,
)
from ..utils.traversal import traverse_obj
class CWTVIE(InfoExtractor):
IE_NAME = 'cwtv'
_VALID_URL = r'https?://(?:www\.)?cw(?:tv(?:pr)?|seed)\.com/(?:shows/)?(?:[^/]+/)+[^?]*\?.*\b(?:play|watch|guid)=(?P<id>[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})'
_TESTS = [{
'url': 'https://www.cwtv.com/shows/continuum/a-stitch-in-time/?play=9149a1e1-4cb2-46d7-81b2-47d35bbd332b',
'info_dict': {
'id': '9149a1e1-4cb2-46d7-81b2-47d35bbd332b',
'ext': 'mp4',
'title': 'A Stitch in Time',
'description': r're:(?s)City Protective Services officer Kiera Cameron is transported from 2077.+',
'thumbnail': r're:https?://.+\.jpe?g',
'duration': 2632,
'timestamp': 1736928000,
'uploader': 'CWTV',
'chapters': 'count:5',
'series': 'Continuum',
'season_number': 1,
'episode_number': 1,
'age_limit': 14,
'upload_date': '20250115',
'season': 'Season 1',
'episode': 'Episode 1',
},
'params': {
# m3u8 download
'skip_download': True,
},
}, {
'url': 'http://cwtv.com/shows/arrow/legends-of-yesterday/?play=6b15e985-9345-4f60-baf8-56e96be57c63',
'info_dict': {
'id': '6b15e985-9345-4f60-baf8-56e96be57c63',
'ext': 'mp4',
'title': 'Legends of Yesterday',
'description': r're:(?s)Oliver and Barry Allen take Kendra Saunders and Carter Hall to a remote.+',
'duration': 2665,
'series': 'Arrow',
'season_number': 4,
'season': '4',
'episode_number': 8,
'upload_date': '20151203',
'timestamp': 1449122100,
},
'params': {
# m3u8 download
'skip_download': True,
},
'skip': 'redirect to http://cwtv.com/shows/arrow/',
}, {
'url': 'http://www.cwseed.com/shows/whose-line-is-it-anyway/jeff-davis-4/?play=24282b12-ead2-42f2-95ad-26770c2c6088',
'info_dict': {
'id': '24282b12-ead2-42f2-95ad-26770c2c6088',
'ext': 'mp4',
'title': 'Jeff Davis 4',
'description': 'Jeff Davis is back to make you laugh.',
'duration': 1263,
'series': 'Whose Line Is It Anyway?',
'season_number': 11,
'episode_number': 20,
'upload_date': '20151006',
'timestamp': 1444107300,
'age_limit': 14,
'uploader': 'CWTV',
'thumbnail': r're:https?://.+\.jpe?g',
'chapters': 'count:4',
'episode': 'Episode 20',
'season': 'Season 11',
},
'params': {
# m3u8 download
'skip_download': True,
},
}, {
'url': 'http://cwtv.com/thecw/chroniclesofcisco/?play=8adebe35-f447-465f-ab52-e863506ff6d6',
'only_matching': True,
}, {
'url': 'http://cwtvpr.com/the-cw/video?watch=9eee3f60-ef4e-440b-b3b2-49428ac9c54e',
'only_matching': True,
}, {
'url': 'http://cwtv.com/shows/arrow/legends-of-yesterday/?watch=6b15e985-9345-4f60-baf8-56e96be57c63',
'only_matching': True,
}, {
'url': 'http://www.cwtv.com/movies/play/?guid=0a8e8b5b-1356-41d5-9a6a-4eda1a6feb6c',
'only_matching': True,
}]
def _real_extract(self, url):
video_id = self._match_id(url)
data = self._download_json(
f'https://images.cwtv.com/feed/app-2/video-meta/apiversion_22/device_android/guid_{video_id}', video_id)
if traverse_obj(data, 'result') != 'ok':
raise ExtractorError(traverse_obj(data, (('error_msg', 'msg'), {str}, any)), expected=True)
video_data = data['video']
title = video_data['title']
mpx_url = update_url_query(
video_data.get('mpx_url') or f'https://link.theplatform.com/s/cwtv/media/guid/2703454149/{video_id}',
{'formats': 'M3U+none'})
season = str_or_none(video_data.get('season'))
episode = str_or_none(video_data.get('episode'))
if episode and season:
episode = episode[len(season):]
return {
'_type': 'url_transparent',
'id': video_id,
'title': title,
'url': smuggle_url(mpx_url, {'force_smil_url': True}),
'description': video_data.get('description_long'),
'duration': int_or_none(video_data.get('duration_secs')),
'series': video_data.get('series_name'),
'season_number': int_or_none(season),
'episode_number': int_or_none(episode),
'timestamp': parse_iso8601(video_data.get('start_time')),
'age_limit': parse_age_limit(video_data.get('rating')),
'ie_key': 'ThePlatform',
'thumbnail': video_data.get('large_thumbnail'),
}
class CWTVMovieIE(InfoExtractor):
IE_NAME = 'cwtv:movie'
_VALID_URL = r'https?://(?:www\.)?cwtv\.com/shows/(?P<id>[\w-]+)/?\?(?:[^#]+&)?viewContext=Movies'
_TESTS = [{
'url': 'https://www.cwtv.com/shows/the-crush/?viewContext=Movies+Swimlane',
'info_dict': {
'id': '0a8e8b5b-1356-41d5-9a6a-4eda1a6feb6c',
'ext': 'mp4',
'title': 'The Crush',
'upload_date': '20241112',
'description': 'md5:1549acd90dff4a8273acd7284458363e',
'chapters': 'count:9',
'timestamp': 1731398400,
'age_limit': 16,
'duration': 5337,
'series': 'The Crush',
'season': 'Season 1',
'uploader': 'CWTV',
'season_number': 1,
'episode': 'Episode 1',
'episode_number': 1,
'thumbnail': r're:https?://.+\.jpe?g',
},
'params': {
# m3u8 download
'skip_download': True,
},
}]
_UUID_RE = r'[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12}'
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
app_url = (
self._html_search_meta('al:ios:url', webpage, default=None)
or self._html_search_meta('al:android:url', webpage, default=None))
video_id = (
traverse_obj(parse_qs(app_url), ('video_id', 0, {lambda x: re.fullmatch(self._UUID_RE, x)}, 0))
or self._search_regex([
rf'CWTV\.Site\.curPlayingGUID\s*=\s*["\']({self._UUID_RE})',
rf'CWTV\.Site\.viewInAppURL\s*=\s*["\']/shows/[\w-]+/watch-in-app/\?play=({self._UUID_RE})',
], webpage, 'video ID'))
return self.url_result(
f'https://www.cwtv.com/shows/{display_id}/{display_id}/?play={video_id}', CWTVIE, video_id)
plugins/youtube_download/yt_dlp/extractor/dailymail.py
+11
View File
@@ -19,11 +19,22 @@ class DailyMailIE(InfoExtractor):
'ext': 'mp4', 'ext': 'mp4',
'title': 'The Mountain appears in sparkling water ad for \'Heavy Bubbles\'', 'title': 'The Mountain appears in sparkling water ad for \'Heavy Bubbles\'',
'description': 'md5:a93d74b6da172dd5dc4d973e0b766a84', 'description': 'md5:a93d74b6da172dd5dc4d973e0b766a84',
'thumbnail': r're:https?://i\.dailymail\.co\.uk/.+\.jpg',
}, },
}, { }, {
'url': 'http://www.dailymail.co.uk/embed/video/1295863.html', 'url': 'http://www.dailymail.co.uk/embed/video/1295863.html',
'only_matching': True, 'only_matching': True,
}] }]
_WEBPAGE_TESTS = [{
'url': 'https://www.daily-news.gr/lifestyle/%ce%b7-%cf%84%cf%81%ce%b1%ce%b3%ce%bf%cf%85%ce%b4%ce%af%cf%83%cf%84%cf%81%ce%b9%ce%b1-jessie-j-%ce%bc%ce%bf%ce%b9%cf%81%ce%ac%cf%83%cf%84%ce%b7%ce%ba%ce%b5-%cf%83%cf%85%ce%b3%ce%ba%ce%bb%ce%bf%ce%bd/',
'info_dict': {
'id': '3463585',
'ext': 'mp4',
'title': 'Jessie J reveals she has undergone surgery as she shares clips',
'description': 'md5:9fa9a25feca5b656b0b4a39c922fad1e',
'thumbnail': r're:https?://i\.dailymail\.co\.uk/.+\.jpg',
},
}]
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
plugins/youtube_download/yt_dlp/extractor/dailymotion.py
+34 -30
View File
@@ -119,13 +119,14 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
_EMBED_REGEX = [rf'(?ix)<(?:(?:embed|iframe)[^>]+?src=|input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=)["\'](?P<url>{_VALID_URL[5:]})'] _EMBED_REGEX = [rf'(?ix)<(?:(?:embed|iframe)[^>]+?src=|input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=)["\'](?P<url>{_VALID_URL[5:]})']
_TESTS = [{ _TESTS = [{
'url': 'http://www.dailymotion.com/video/x5kesuj_office-christmas-party-review-jason-bateman-olivia-munn-t-j-miller_news', 'url': 'http://www.dailymotion.com/video/x5kesuj_office-christmas-party-review-jason-bateman-olivia-munn-t-j-miller_news',
'md5': '074b95bdee76b9e3654137aee9c79dfe',
'info_dict': { 'info_dict': {
'id': 'x5kesuj', 'id': 'x5kesuj',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Office Christmas Party Review – Jason Bateman, Olivia Munn, T.J. Miller', 'title': 'Office Christmas Party Review – Jason Bateman, Olivia Munn, T.J. Miller',
'description': 'Office Christmas Party Review - Jason Bateman, Olivia Munn, T.J. Miller', 'description': 'Office Christmas Party Review - Jason Bateman, Olivia Munn, T.J. Miller',
'duration': 187, 'duration': 187,
'tags': 'count:5',
'thumbnail': r're:https?://s[12]\.dmcdn\.net/v/.+',
'timestamp': 1493651285, 'timestamp': 1493651285,
'upload_date': '20170501', 'upload_date': '20170501',
'uploader': 'Deadline', 'uploader': 'Deadline',
@@ -133,18 +134,17 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
'age_limit': 0, 'age_limit': 0,
'view_count': int, 'view_count': int,
'like_count': int, 'like_count': int,
'tags': ['hollywood', 'celeb', 'celebrity', 'movies', 'red carpet'],
'thumbnail': r're:https://(?:s[12]\.)dmcdn\.net/v/K456B1cmt4ZcZ9KiM/x1080',
}, },
}, { }, {
'url': 'https://geo.dailymotion.com/player.html?video=x89eyek&mute=true', 'url': 'https://geo.dailymotion.com/player.html?video=x89eyek&mute=true',
'md5': 'e2f9717c6604773f963f069ca53a07f8',
'info_dict': { 'info_dict': {
'id': 'x89eyek', 'id': 'x89eyek',
'ext': 'mp4', 'ext': 'mp4',
'title': "En quête d'esprit du 27/03/2022", 'title': 'En quête d\'esprit du 27/03/2022',
'description': 'md5:66542b9f4df2eb23f314fc097488e553', 'description': 'md5:66542b9f4df2eb23f314fc097488e553',
'duration': 2756, 'duration': 2756,
'tags': 'count:1',
'thumbnail': r're:https?://s[12]\.dmcdn\.net/v/.+',
'timestamp': 1648383669, 'timestamp': 1648383669,
'upload_date': '20220327', 'upload_date': '20220327',
'uploader': 'CNEWS', 'uploader': 'CNEWS',
@@ -152,8 +152,6 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
'age_limit': 0, 'age_limit': 0,
'view_count': int, 'view_count': int,
'like_count': int, 'like_count': int,
'tags': ['en_quete_d_esprit'],
'thumbnail': r're:https://(?:s[12]\.)dmcdn\.net/v/Tncwi1clTH6StrxMP/x1080',
}, },
}, { }, {
'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames', 'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames',
@@ -163,8 +161,8 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
'ext': 'mp4', 'ext': 'mp4',
'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News', 'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
'description': 'Several come bundled with the Steam Controller.', 'description': 'Several come bundled with the Steam Controller.',
'thumbnail': r're:^https?:.*\.(?:jpg|png)$',
'duration': 74, 'duration': 74,
'thumbnail': r're:https?://s[12]\.dmcdn\.net/v/.+',
'timestamp': 1425657362, 'timestamp': 1425657362,
'upload_date': '20150306', 'upload_date': '20150306',
'uploader': 'IGN', 'uploader': 'IGN',
@@ -173,20 +171,6 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
'view_count': int, 'view_count': int,
}, },
'skip': 'video gone', 'skip': 'video gone',
}, {
# Vevo video
'url': 'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi',
'info_dict': {
'title': 'Roar (Official)',
'id': 'USUV71301934',
'ext': 'mp4',
'uploader': 'Katy Perry',
'upload_date': '20130905',
},
'params': {
'skip_download': True,
},
'skip': 'VEVO is only available in some countries',
}, { }, {
# age-restricted video # age-restricted video
'url': 'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband', 'url': 'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband',
@@ -259,9 +243,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
'uploader_id': 'x2vtgmm', 'uploader_id': 'x2vtgmm',
'age_limit': 0, 'age_limit': 0,
'tags': [], 'tags': [],
'thumbnail': r're:https?://s[12]\.dmcdn\.net/v/.+',
'view_count': int, 'view_count': int,
'like_count': int, 'like_count': int,
'thumbnail': r're:https://\w+.dmcdn.net/v/WnEY61cmvMxt2Fi6d/x1080',
}, },
}, { }, {
# https://geo.dailymotion.com/player/xf7zn.html?playlist=x7wdsj # https://geo.dailymotion.com/player/xf7zn.html?playlist=x7wdsj
@@ -276,18 +260,18 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
'info_dict': { 'info_dict': {
'id': 'x8u4owg', 'id': 'x8u4owg',
'ext': 'mp4', 'ext': 'mp4',
'description': 'À bord du « véloto », l’alternative à la voiture pour la campagne',
'like_count': int, 'like_count': int,
'uploader': 'Le Parisien', 'uploader': 'Le Parisien',
'thumbnail': 'https://www.leparisien.fr/resizer/ho_GwveeYftNkLwg_cEta--5Bv4=/1200x675/cloudfront-eu-central-1.images.arcpublishing.com/leparisien/BFXJNEBN75EUNHGYJLORUC3TX4.jpg',
'upload_date': '20240309', 'upload_date': '20240309',
'view_count': int, 'view_count': int,
'tags': 'count:7',
'thumbnail': r're:https?://www\.leparisien\.fr/.+\.jpg',
'timestamp': 1709997866, 'timestamp': 1709997866,
'age_limit': 0, 'age_limit': 0,
'uploader_id': 'x32f7b', 'uploader_id': 'x32f7b',
'title': 'VIDÉO. Le «\xa0véloto\xa0», la voiture à pédales qui aimerait se faire une place sur les routes', 'title': 'VIDÉO. Le «\xa0véloto\xa0», la voiture à pédales qui aimerait se faire une place sur les routes',
'duration': 428.0, 'duration': 428.0,
'description': 'À bord du « véloto », l’alternative à la voiture pour la campagne',
'tags': ['biclou', 'vélo', 'véloto', 'campagne', 'voiture', 'environnement', 'véhicules intermédiaires'],
}, },
}, { }, {
# https://geo.dailymotion.com/player/xry80.html?video=x8vu47w # https://geo.dailymotion.com/player/xry80.html?video=x8vu47w
@@ -297,9 +281,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
'ext': 'mp4', 'ext': 'mp4',
'like_count': int, 'like_count': int,
'uploader': 'Metatube', 'uploader': 'Metatube',
'thumbnail': r're:https://\w+.dmcdn.net/v/W1G_S1coGSFTfkTeR/x1080',
'upload_date': '20240326', 'upload_date': '20240326',
'view_count': int, 'view_count': int,
'thumbnail': r're:https?://s[12]\.dmcdn\.net/v/.+',
'timestamp': 1711496732, 'timestamp': 1711496732,
'age_limit': 0, 'age_limit': 0,
'uploader_id': 'x2xpy74', 'uploader_id': 'x2xpy74',
@@ -308,6 +292,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
'description': 'Que lindura', 'description': 'Que lindura',
'tags': [], 'tags': [],
}, },
'skip': 'Invalid URL',
}, { }, {
# //geo.dailymotion.com/player/xysxq.html?video=k2Y4Mjp7krAF9iCuINM # //geo.dailymotion.com/player/xysxq.html?video=k2Y4Mjp7krAF9iCuINM
'url': 'https://lcp.fr/programmes/avant-la-catastrophe-la-naissance-de-la-dictature-nazie-1933-1936-346819', 'url': 'https://lcp.fr/programmes/avant-la-catastrophe-la-naissance-de-la-dictature-nazie-1933-1936-346819',
@@ -322,11 +307,30 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
'like_count': int, 'like_count': int,
'age_limit': 0, 'age_limit': 0,
'duration': 3220, 'duration': 3220,
'thumbnail': 'https://s1.dmcdn.net/v/Xvumk1djJBUZfjj2a/x1080',
'tags': [], 'tags': [],
'thumbnail': r're:https?://s[12]\.dmcdn\.net/v/.+',
'timestamp': 1739919947, 'timestamp': 1739919947,
'upload_date': '20250218', 'upload_date': '20250218',
}, },
'skip': 'Invalid URL',
}, {
'url': 'https://forum.ionicframework.com/t/ionic-2-jw-player-dailymotion-player/83248',
'info_dict': {
'id': 'xwr14q',
'ext': 'mp4',
'title': 'Macklemore & Ryan Lewis - Thrift Shop (feat. Wanz)',
'age_limit': 0,
'description': 'md5:47fbe168b5a6ddc4a205e20dd6c841b2',
'duration': 234,
'like_count': int,
'tags': 'count:5',
'thumbnail': r're:https?://s[12]\.dmcdn\.net/v/.+',
'timestamp': 1358177670,
'upload_date': '20130114',
'uploader': 'Macklemore Official',
'uploader_id': 'x19qlwr',
'view_count': int,
},
}] }]
_GEO_BYPASS = False _GEO_BYPASS = False
_COMMON_MEDIA_FIELDS = '''description _COMMON_MEDIA_FIELDS = '''description
@@ -540,7 +544,7 @@ class DailymotionSearchIE(DailymotionPlaylistBaseIE):
'id': 'king of turtles', 'id': 'king of turtles',
'title': 'king of turtles', 'title': 'king of turtles',
}, },
'playlist_mincount': 90, 'playlist_mincount': 0,
}] }]
_SEARCH_QUERY = 'query SEARCH_QUERY( $query: String! $page: Int $limit: Int ) { search { videos( query: $query first: $limit page: $page ) { edges { node { xid } } } } } ' _SEARCH_QUERY = 'query SEARCH_QUERY( $query: String! $page: Int $limit: Int ) { search { videos( query: $query first: $limit page: $page ) { edges { node { xid } } } } } '
@@ -584,7 +588,7 @@ class DailymotionUserIE(DailymotionPlaylistBaseIE):
'info_dict': { 'info_dict': {
'id': 'nqtv', 'id': 'nqtv',
}, },
'playlist_mincount': 152, 'playlist_mincount': 148,
}, { }, {
'url': 'http://www.dailymotion.com/user/UnderProject', 'url': 'http://www.dailymotion.com/user/UnderProject',
'info_dict': { 'info_dict': {
plugins/youtube_download/yt_dlp/extractor/dangalplay.py
+20 -7
View File
@@ -11,8 +11,14 @@ from ..utils.traversal import traverse_obj
class DangalPlayBaseIE(InfoExtractor): class DangalPlayBaseIE(InfoExtractor):
_NETRC_MACHINE = 'dangalplay' _NETRC_MACHINE = 'dangalplay'
_REGION = 'IN'
_OTV_USER_ID = None _OTV_USER_ID = None
_LOGIN_HINT = 'Pass credentials as -u "token" -p "USER_ID" where USER_ID is the `otv_user_id` in browser local storage' _LOGIN_HINT = (
'Pass credentials as -u "token" -p "USER_ID" '
'(where USER_ID is the value of "otv_user_id" in your browser local storage). '
'Your login region can be optionally suffixed to the username as @REGION '
'(where REGION is the two-letter "region" code found in your browser local storage), '
'e.g.: -u "token@IN" -p "USER_ID"')
_API_BASE = 'https://ottapi.dangalplay.com' _API_BASE = 'https://ottapi.dangalplay.com'
_AUTH_TOKEN = 'jqeGWxRKK7FK5zEk3xCM' # from https://www.dangalplay.com/main.48ad19e24eb46acccef3.js _AUTH_TOKEN = 'jqeGWxRKK7FK5zEk3xCM' # from https://www.dangalplay.com/main.48ad19e24eb46acccef3.js
_SECRET_KEY = 'f53d31a4377e4ef31fa0' # same as above _SECRET_KEY = 'f53d31a4377e4ef31fa0' # same as above
@@ -20,8 +26,12 @@ class DangalPlayBaseIE(InfoExtractor):
def _perform_login(self, username, password): def _perform_login(self, username, password):
if self._OTV_USER_ID: if self._OTV_USER_ID:
return return
if username != 'token' or not re.fullmatch(r'[\da-f]{32}', password): mobj = re.fullmatch(r'token(?:@(?P<region>[A-Z]{2}))?', username)
if not mobj or not re.fullmatch(r'[\da-f]{32}', password):
raise ExtractorError(self._LOGIN_HINT, expected=True) raise ExtractorError(self._LOGIN_HINT, expected=True)
if region := mobj.group('region'):
self._REGION = region
self.write_debug(f'Setting login region to "{self._REGION}"')
self._OTV_USER_ID = password self._OTV_USER_ID = password
def _real_initialize(self): def _real_initialize(self):
@@ -52,7 +62,7 @@ class DangalPlayBaseIE(InfoExtractor):
f'{self._API_BASE}/{path}', display_id, note, fatal=fatal, f'{self._API_BASE}/{path}', display_id, note, fatal=fatal,
headers={'Accept': 'application/json'}, query={ headers={'Accept': 'application/json'}, query={
'auth_token': self._AUTH_TOKEN, 'auth_token': self._AUTH_TOKEN,
'region': 'IN', 'region': self._REGION,
**query, **query,
}) })
@@ -106,7 +116,7 @@ class DangalPlayIE(DangalPlayBaseIE):
'catalog_id': catalog_id, 'catalog_id': catalog_id,
'content_id': content_id, 'content_id': content_id,
'category': '', 'category': '',
'region': 'IN', 'region': self._REGION,
'auth_token': self._AUTH_TOKEN, 'auth_token': self._AUTH_TOKEN,
'id': self._OTV_USER_ID, 'id': self._OTV_USER_ID,
'md5': hashlib.md5(unhashed.encode()).hexdigest(), 'md5': hashlib.md5(unhashed.encode()).hexdigest(),
@@ -129,11 +139,14 @@ class DangalPlayIE(DangalPlayBaseIE):
except ExtractorError as e: except ExtractorError as e:
if isinstance(e.cause, HTTPError) and e.cause.status == 422: if isinstance(e.cause, HTTPError) and e.cause.status == 422:
error_info = traverse_obj(e.cause.response.read().decode(), ({json.loads}, 'error', {dict})) or {} error_info = traverse_obj(e.cause.response.read().decode(), ({json.loads}, 'error', {dict})) or {}
if error_info.get('code') == '1016': error_code = error_info.get('code')
if error_code == '1016':
self.raise_login_required( self.raise_login_required(
f'Your token has expired or is invalid. {self._LOGIN_HINT}', method=None) f'Your token has expired or is invalid. {self._LOGIN_HINT}', method=None)
elif msg := error_info.get('message'): elif error_code == '4028':
raise ExtractorError(msg) self.raise_login_required(
f'Your login region is unspecified or incorrect. {self._LOGIN_HINT}', method=None)
raise ExtractorError(join_nonempty(error_code, error_info.get('message'), delim=': '))
raise raise
m3u8_url = traverse_obj(details, ( m3u8_url = traverse_obj(details, (
plugins/youtube_download/yt_dlp/extractor/dbtv.py
+16 -2
View File
@@ -12,13 +12,13 @@ class DBTVIE(InfoExtractor):
'ext': 'mp4', 'ext': 'mp4',
'title': 'Skulle teste ut fornøyelsespark, men kollegaen var bare opptatt av bikinikroppen', 'title': 'Skulle teste ut fornøyelsespark, men kollegaen var bare opptatt av bikinikroppen',
'description': 'md5:49cc8370e7d66e8a2ef15c3b4631fd3f', 'description': 'md5:49cc8370e7d66e8a2ef15c3b4631fd3f',
'thumbnail': r're:https?://.*\.jpg', 'thumbnail': r're:https?://.+\.jpg',
'upload_date': '20160916', 'upload_date': '20160916',
'duration': 69, 'duration': 69,
'uploader_id': 'UCk5pvsyZJoYJBd7_oFPTlRQ', 'uploader_id': 'UCk5pvsyZJoYJBd7_oFPTlRQ',
'uploader': 'Dagbladet', 'uploader': 'Dagbladet',
}, },
'add_ie': ['Youtube'], 'skip': 'Invalid URL',
}, { }, {
'url': 'https://www.dagbladet.no/video/embed/xlGmyIeN9Jo/?autoplay=false', 'url': 'https://www.dagbladet.no/video/embed/xlGmyIeN9Jo/?autoplay=false',
'only_matching': True, 'only_matching': True,
@@ -26,6 +26,20 @@ class DBTVIE(InfoExtractor):
'url': 'https://www.dagbladet.no/video/truer-iran-bor-passe-dere/PalfB2Cw', 'url': 'https://www.dagbladet.no/video/truer-iran-bor-passe-dere/PalfB2Cw',
'only_matching': True, 'only_matching': True,
}] }]
_WEBPAGE_TESTS = [{
# FIXME: Embed detection
'url': 'https://www.dagbladet.no/nyheter/rekordstort-russisk-angrep/83325693',
'info_dict': {
'id': '1HW7fYry',
'ext': 'mp4',
'title': 'Putin taler - så skjer dette',
'description': 'md5:3e8bacee33de861a9663d9a3fcc54e5e',
'display_id': 'putin-taler-sa-skjer-dette',
'thumbnail': r're:https?://cdn\.jwplayer\.com/v2/media/.+',
'timestamp': 1751043600,
'upload_date': '20250627',
},
}]
def _real_extract(self, url): def _real_extract(self, url):
display_id, video_id = self._match_valid_url(url).groups() display_id, video_id = self._match_valid_url(url).groups()
plugins/youtube_download/yt_dlp/extractor/digitalconcerthall.py
+13 -5
View File
@@ -4,6 +4,7 @@ from .common import InfoExtractor
from ..networking.exceptions import HTTPError from ..networking.exceptions import HTTPError
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
determine_ext,
jwt_decode_hs256, jwt_decode_hs256,
parse_codecs, parse_codecs,
try_get, try_get,
@@ -222,11 +223,18 @@ class DigitalConcertHallIE(InfoExtractor):
raise raise
formats = [] formats = []
for m3u8_url in traverse_obj(stream_info, ('channel', ..., 'stream', ..., 'url', {url_or_none})): for fmt_url in traverse_obj(stream_info, ('channel', ..., 'stream', ..., 'url', {url_or_none})):
formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', m3u8_id='hls', fatal=False)) ext = determine_ext(fmt_url)
for fmt in formats: if ext == 'm3u8':
if fmt.get('format_note') and fmt.get('vcodec') == 'none': fmts = self._extract_m3u8_formats(fmt_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
fmt.update(parse_codecs(fmt['format_note'])) for fmt in fmts:
if fmt.get('format_note') and fmt.get('vcodec') == 'none':
fmt.update(parse_codecs(fmt['format_note']))
formats.extend(fmts)
elif ext == 'mpd':
formats.extend(self._extract_mpd_formats(fmt_url, video_id, mpd_id='dash', fatal=False))
else:
self.report_warning(f'Skipping unsupported format extension "{ext}"')
yield { yield {
'id': video_id, 'id': video_id,
plugins/youtube_download/yt_dlp/extractor/douyutv.py
+1 -1
View File
@@ -206,7 +206,7 @@ class DouyuTVIE(DouyuBaseIE):
'is_live': True, 'is_live': True,
**traverse_obj(room, { **traverse_obj(room, {
'display_id': ('url', {str}, {lambda i: i[1:]}), 'display_id': ('url', {str}, {lambda i: i[1:]}),
'title': ('room_name', {unescapeHTML}), 'title': ('room_name', {str}, {unescapeHTML}),
'description': ('show_details', {str}), 'description': ('show_details', {str}),
'uploader': ('nickname', {str}), 'uploader': ('nickname', {str}),
'thumbnail': ('room_src', {url_or_none}), 'thumbnail': ('room_src', {url_or_none}),
plugins/youtube_download/yt_dlp/extractor/dreisat.py
+1 -1
View File
@@ -64,7 +64,7 @@ class DreiSatIE(ZDFBaseIE):
'title': 'dein buch - Das Beste von der Leipziger Buchmesse 2025 - Teil 1', 'title': 'dein buch - Das Beste von der Leipziger Buchmesse 2025 - Teil 1',
'description': 'md5:bae51bfc22f15563ce3acbf97d2e8844', 'description': 'md5:bae51bfc22f15563ce3acbf97d2e8844',
'duration': 5399.0, 'duration': 5399.0,
'thumbnail': 'https://www.3sat.de/assets/buchmesse-kerkeling-100~original?cb=1743329640903', 'thumbnail': 'https://www.3sat.de/assets/buchmesse-kerkeling-100~original?cb=1747256996338',
'chapters': 'count:24', 'chapters': 'count:24',
'episode': 'dein buch - Das Beste von der Leipziger Buchmesse 2025 - Teil 1', 'episode': 'dein buch - Das Beste von der Leipziger Buchmesse 2025 - Teil 1',
'episode_id': 'POS_1ef236cc-b390-401e-acd0-4fb4b04315fb', 'episode_id': 'POS_1ef236cc-b390-401e-acd0-4fb4b04315fb',
plugins/youtube_download/yt_dlp/extractor/dropout.py
+26 -26
View File
@@ -18,15 +18,15 @@ from ..utils import (
class DropoutIE(InfoExtractor): class DropoutIE(InfoExtractor):
_LOGIN_URL = 'https://www.dropout.tv/login' _LOGIN_URL = 'https://watch.dropout.tv/login'
_NETRC_MACHINE = 'dropout' _NETRC_MACHINE = 'dropout'
_VALID_URL = r'https?://(?:www\.)?dropout\.tv/(?:[^/]+/)*videos/(?P<id>[^/]+)/?$' _VALID_URL = r'https?://(?:watch\.)?dropout\.tv/(?:[^/?#]+/)*videos/(?P<id>[^/?#]+)/?(?:[?#]|$)'
_TESTS = [ _TESTS = [
{ {
'url': 'https://www.dropout.tv/game-changer/season:2/videos/yes-or-no', 'url': 'https://watch.dropout.tv/game-changer/season:2/videos/yes-or-no',
'note': 'Episode in a series', 'note': 'Episode in a series',
'md5': '5e000fdfd8d8fa46ff40456f1c2af04a', 'md5': '4b76963f904f8bc4ba22dcf0e66ada06',
'info_dict': { 'info_dict': {
'id': '738153', 'id': '738153',
'display_id': 'yes-or-no', 'display_id': 'yes-or-no',
@@ -45,35 +45,35 @@ class DropoutIE(InfoExtractor):
'uploader_url': 'https://vimeo.com/user80538407', 'uploader_url': 'https://vimeo.com/user80538407',
'uploader': 'OTT Videos', 'uploader': 'OTT Videos',
}, },
'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'], 'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest', 'Failed to parse XML: not well-formed'],
}, },
{ {
'url': 'https://www.dropout.tv/dimension-20-fantasy-high/season:1/videos/episode-1', 'url': 'https://watch.dropout.tv/tablepop-presents-megadungeon-live/season:1/videos/enter-through-the-gift-shop',
'note': 'Episode in a series (missing release_date)', 'note': 'Episode in a series (missing release_date)',
'md5': '712caf7c191f1c47c8f1879520c2fa5c', 'md5': 'b08fb03050585ea25cd7ee092db9134c',
'info_dict': { 'info_dict': {
'id': '320562', 'id': '624270',
'display_id': 'episode-1', 'display_id': 'enter-through-the-gift-shop',
'ext': 'mp4', 'ext': 'mp4',
'title': 'The Beginning Begins', 'title': 'Enter Through the Gift Shop',
'description': 'The cast introduces their PCs, including a neurotic elf, a goblin PI, and a corn-worshipping cleric.', 'description': 'A new adventuring party explores a gift shop and runs into a friendly orc -- and some angry goblins.',
'thumbnail': 'https://vhx.imgix.net/chuncensoredstaging/assets/4421ed0d-f630-4c88-9004-5251b2b8adfa.jpg', 'thumbnail': 'https://vhx.imgix.net/chuncensoredstaging/assets/a1d876c3-3dee-4cd0-87c6-27a851b1d0ec.jpg',
'series': 'Dimension 20: Fantasy High', 'series': 'TablePop Presents: MEGADUNGEON LIVE!',
'season_number': 1, 'season_number': 1,
'season': 'Season 1', 'season': 'Season 1',
'episode_number': 1, 'episode_number': 1,
'episode': 'The Beginning Begins', 'episode': 'Enter Through the Gift Shop',
'duration': 6838, 'duration': 7101,
'uploader_id': 'user80538407', 'uploader_id': 'user80538407',
'uploader_url': 'https://vimeo.com/user80538407', 'uploader_url': 'https://vimeo.com/user80538407',
'uploader': 'OTT Videos', 'uploader': 'OTT Videos',
}, },
'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'], 'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest', 'Failed to parse XML: not well-formed'],
}, },
{ {
'url': 'https://www.dropout.tv/videos/misfits-magic-holiday-special', 'url': 'https://watch.dropout.tv/videos/misfits-magic-holiday-special',
'note': 'Episode not in a series', 'note': 'Episode not in a series',
'md5': 'c30fa18999c5880d156339f13c953a26', 'md5': '1e6428f7756b02c93b573d39ddd789fe',
'info_dict': { 'info_dict': {
'id': '1915774', 'id': '1915774',
'display_id': 'misfits-magic-holiday-special', 'display_id': 'misfits-magic-holiday-special',
@@ -87,7 +87,7 @@ class DropoutIE(InfoExtractor):
'uploader_url': 'https://vimeo.com/user80538407', 'uploader_url': 'https://vimeo.com/user80538407',
'uploader': 'OTT Videos', 'uploader': 'OTT Videos',
}, },
'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'], 'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest', 'Failed to parse XML: not well-formed'],
}, },
] ]
@@ -125,7 +125,7 @@ class DropoutIE(InfoExtractor):
display_id = self._match_id(url) display_id = self._match_id(url)
webpage = None webpage = None
if self._get_cookies('https://www.dropout.tv').get('_session'): if self._get_cookies('https://watch.dropout.tv').get('_session'):
webpage = self._download_webpage(url, display_id) webpage = self._download_webpage(url, display_id)
if not webpage or '<div id="watch-unauthorized"' in webpage: if not webpage or '<div id="watch-unauthorized"' in webpage:
login_err = self._login(display_id) login_err = self._login(display_id)
@@ -148,7 +148,7 @@ class DropoutIE(InfoExtractor):
return { return {
'_type': 'url_transparent', '_type': 'url_transparent',
'ie_key': VHXEmbedIE.ie_key(), 'ie_key': VHXEmbedIE.ie_key(),
'url': VHXEmbedIE._smuggle_referrer(embed_url, 'https://www.dropout.tv'), 'url': VHXEmbedIE._smuggle_referrer(embed_url, 'https://watch.dropout.tv'),
'id': self._search_regex(r'embed\.vhx\.tv/videos/(.+?)\?', embed_url, 'id'), 'id': self._search_regex(r'embed\.vhx\.tv/videos/(.+?)\?', embed_url, 'id'),
'display_id': display_id, 'display_id': display_id,
'title': title, 'title': title,
@@ -167,10 +167,10 @@ class DropoutIE(InfoExtractor):
class DropoutSeasonIE(InfoExtractor): class DropoutSeasonIE(InfoExtractor):
_PAGE_SIZE = 24 _PAGE_SIZE = 24
_VALID_URL = r'https?://(?:www\.)?dropout\.tv/(?P<id>[^\/$&?#]+)(?:/?$|/season:(?P<season>[0-9]+)/?$)' _VALID_URL = r'https?://(?:watch\.)?dropout\.tv/(?P<id>[^\/$&?#]+)(?:/?$|/season:(?P<season>[0-9]+)/?$)'
_TESTS = [ _TESTS = [
{ {
'url': 'https://www.dropout.tv/dimension-20-fantasy-high/season:1', 'url': 'https://watch.dropout.tv/dimension-20-fantasy-high/season:1',
'note': 'Multi-season series with the season in the url', 'note': 'Multi-season series with the season in the url',
'playlist_count': 24, 'playlist_count': 24,
'info_dict': { 'info_dict': {
@@ -179,7 +179,7 @@ class DropoutSeasonIE(InfoExtractor):
}, },
}, },
{ {
'url': 'https://www.dropout.tv/dimension-20-fantasy-high', 'url': 'https://watch.dropout.tv/dimension-20-fantasy-high',
'note': 'Multi-season series with the season not in the url', 'note': 'Multi-season series with the season not in the url',
'playlist_count': 24, 'playlist_count': 24,
'info_dict': { 'info_dict': {
@@ -188,7 +188,7 @@ class DropoutSeasonIE(InfoExtractor):
}, },
}, },
{ {
'url': 'https://www.dropout.tv/dimension-20-shriek-week', 'url': 'https://watch.dropout.tv/dimension-20-shriek-week',
'note': 'Single-season series', 'note': 'Single-season series',
'playlist_count': 4, 'playlist_count': 4,
'info_dict': { 'info_dict': {
@@ -197,7 +197,7 @@ class DropoutSeasonIE(InfoExtractor):
}, },
}, },
{ {
'url': 'https://www.dropout.tv/breaking-news-no-laugh-newsroom/season:3', 'url': 'https://watch.dropout.tv/breaking-news-no-laugh-newsroom/season:3',
'note': 'Multi-season series with season in the url that requires pagination', 'note': 'Multi-season series with season in the url that requires pagination',
'playlist_count': 25, 'playlist_count': 25,
'info_dict': { 'info_dict': {
plugins/youtube_download/yt_dlp/extractor/dtube.py
+1 -2
View File
@@ -1,5 +1,4 @@
import json import json
import socket
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import (
@@ -56,7 +55,7 @@ class DTubeIE(InfoExtractor):
try: try:
self.to_screen(f'{video_id}: Checking {format_id} video format URL') self.to_screen(f'{video_id}: Checking {format_id} video format URL')
self._downloader._opener.open(video_url, timeout=5).close() self._downloader._opener.open(video_url, timeout=5).close()
except socket.timeout: except TimeoutError:
self.to_screen( self.to_screen(
f'{video_id}: {format_id} URL is invalid, skipping') f'{video_id}: {format_id} URL is invalid, skipping')
continue continue
plugins/youtube_download/yt_dlp/extractor/eagleplatform.py
-215
View File
@@ -1,215 +0,0 @@
import functools
import re
from .common import InfoExtractor
from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
smuggle_url,
unsmuggle_url,
url_or_none,
)
class EaglePlatformIE(InfoExtractor):
_VALID_URL = r'''(?x)
(?:
eagleplatform:(?P<custom_host>[^/]+):|
https?://(?P<host>.+?\.media\.eagleplatform\.com)/index/player\?.*\brecord_id=
)
(?P<id>\d+)
'''
_EMBED_REGEX = [r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//.+?\.media\.eagleplatform\.com/index/player\?.+?)\1']
_TESTS = [{
# http://lenta.ru/news/2015/03/06/navalny/
'url': 'http://lentaru.media.eagleplatform.com/index/player?player=new&record_id=227304&player_template_id=5201',
# Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
'info_dict': {
'id': '227304',
'ext': 'mp4',
'title': 'Навальный вышел на свободу',
'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 87,
'view_count': int,
'age_limit': 0,
},
}, {
# http://muz-tv.ru/play/7129/
# http://media.clipyou.ru/index/player?record_id=12820&width=730&height=415&autoplay=true
'url': 'eagleplatform:media.clipyou.ru:12820',
'md5': '358597369cf8ba56675c1df15e7af624',
'info_dict': {
'id': '12820',
'ext': 'mp4',
'title': "'O Sole Mio",
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 216,
'view_count': int,
},
'skip': 'Georestricted',
}, {
# referrer protected video (https://tvrain.ru/lite/teleshow/kak_vse_nachinalos/namin-418921/)
'url': 'eagleplatform:tvrainru.media.eagleplatform.com:582306',
'only_matching': True,
}]
@classmethod
def _extract_embed_urls(cls, url, webpage):
add_referer = functools.partial(smuggle_url, data={'referrer': url})
res = tuple(super()._extract_embed_urls(url, webpage))
if res:
return map(add_referer, res)
PLAYER_JS_RE = r'''
<script[^>]+
src=(?P<qjs>["\'])(?:https?:)?//(?P<host>(?:(?!(?P=qjs)).)+\.media\.eagleplatform\.com)/player/player\.js(?P=qjs)
.+?
'''
# "Basic usage" embedding (see http://dultonmedia.github.io/eplayer/)
mobj = re.search(
rf'''(?xs)
{PLAYER_JS_RE}
<div[^>]+
class=(?P<qclass>["\'])eagleplayer(?P=qclass)[^>]+
data-id=["\'](?P<id>\d+)
''', webpage)
if mobj is not None:
return [add_referer('eagleplatform:{host}:{id}'.format(**mobj.groupdict()))]
# Generalization of "Javascript code usage", "Combined usage" and
# "Usage without attaching to DOM" embeddings (see
# http://dultonmedia.github.io/eplayer/)
mobj = re.search(
r'''(?xs)
%s
<script>
.+?
new\s+EaglePlayer\(
(?:[^,]+\s*,\s*)?
{
.+?
\bid\s*:\s*["\']?(?P<id>\d+)
.+?
}
\s*\)
.+?
</script>
''' % PLAYER_JS_RE, webpage) # noqa: UP031
if mobj is not None:
return [add_referer('eagleplatform:{host}:{id}'.format(**mobj.groupdict()))]
@staticmethod
def _handle_error(response):
status = int_or_none(response.get('status', 200))
if status != 200:
raise ExtractorError(' '.join(response['errors']), expected=True)
def _download_json(self, url_or_request, video_id, *args, **kwargs):
try:
response = super()._download_json(
url_or_request, video_id, *args, **kwargs)
except ExtractorError as ee:
if isinstance(ee.cause, HTTPError):
response = self._parse_json(ee.cause.response.read().decode('utf-8'), video_id)
self._handle_error(response)
raise
return response
def _get_video_url(self, url_or_request, video_id, note='Downloading JSON metadata'):
return self._download_json(url_or_request, video_id, note)['data'][0]
def _real_extract(self, url):
url, smuggled_data = unsmuggle_url(url, {})
mobj = self._match_valid_url(url)
host, video_id = mobj.group('custom_host') or mobj.group('host'), mobj.group('id')
headers = {}
query = {
'id': video_id,
}
referrer = smuggled_data.get('referrer')
if referrer:
headers['Referer'] = referrer
query['referrer'] = referrer
player_data = self._download_json(
f'http://{host}/api/player_data', video_id,
headers=headers, query=query)
media = player_data['data']['playlist']['viewports'][0]['medialist'][0]
title = media['title']
description = media.get('description')
thumbnail = self._proto_relative_url(media.get('snapshot'), 'http:')
duration = int_or_none(media.get('duration'))
view_count = int_or_none(media.get('views'))
age_restriction = media.get('age_restriction')
age_limit = None
if age_restriction:
age_limit = 0 if age_restriction == 'allow_all' else 18
secure_m3u8 = self._proto_relative_url(media['sources']['secure_m3u8']['auto'], 'http:')
formats = []
m3u8_url = self._get_video_url(secure_m3u8, video_id, 'Downloading m3u8 JSON')
m3u8_formats = self._extract_m3u8_formats(
m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
m3u8_id='hls', fatal=False)
formats.extend(m3u8_formats)
m3u8_formats_dict = {}
for f in m3u8_formats:
if f.get('height') is not None:
m3u8_formats_dict[f['height']] = f
mp4_data = self._download_json(
# Secure mp4 URL is constructed according to Player.prototype.mp4 from
# http://lentaru.media.eagleplatform.com/player/player.js
re.sub(r'm3u8|hlsvod|hls|f4m', 'mp4s', secure_m3u8),
video_id, 'Downloading mp4 JSON', fatal=False)
if mp4_data:
for format_id, format_url in mp4_data.get('data', {}).items():
if not url_or_none(format_url):
continue
height = int_or_none(format_id)
if height is not None and m3u8_formats_dict.get(height):
f = m3u8_formats_dict[height].copy()
f.update({
'format_id': f['format_id'].replace('hls', 'http'),
'protocol': 'http',
})
else:
f = {
'format_id': f'http-{format_id}',
'height': int_or_none(format_id),
}
f['url'] = format_url
formats.append(f)
return {
'id': video_id,
'title': title,
'description': description,
'thumbnail': thumbnail,
'duration': duration,
'view_count': view_count,
'age_limit': age_limit,
'formats': formats,
}
class ClipYouEmbedIE(InfoExtractor):
_VALID_URL = False
@classmethod
def _extract_embed_urls(cls, url, webpage):
mobj = re.search(
r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
if mobj is not None:
yield smuggle_url('eagleplatform:{host}:{id}'.format(**mobj.groupdict()), {'referrer': url})
plugins/youtube_download/yt_dlp/extractor/ertgr.py
+24 -15
View File
@@ -64,14 +64,12 @@ class ERTFlixCodenameIE(ERTFlixBaseIE):
_VALID_URL = r'ertflix:(?P<id>[\w-]+)' _VALID_URL = r'ertflix:(?P<id>[\w-]+)'
_TESTS = [{ _TESTS = [{
'url': 'ertflix:monogramma-praxitelis-tzanoylinos', 'url': 'ertflix:monogramma-praxitelis-tzanoylinos',
'md5': '5b9c2cd171f09126167e4082fc1dd0ef',
'info_dict': { 'info_dict': {
'id': 'monogramma-praxitelis-tzanoylinos', 'id': 'monogramma-praxitelis-tzanoylinos',
'ext': 'mp4', 'ext': 'mp4',
'title': 'md5:ef0b439902963d56c43ac83c3f41dd0e', 'title': 'monogramma-praxitelis-tzanoylinos',
}, },
}, }]
]
def _extract_formats_and_subs(self, video_id): def _extract_formats_and_subs(self, video_id):
media_info = self._call_api(video_id, codename=video_id) media_info = self._call_api(video_id, codename=video_id)
@@ -131,13 +129,14 @@ class ERTFlixIE(ERTFlixBaseIE):
'duration': 3166, 'duration': 3166,
'age_limit': 8, 'age_limit': 8,
}, },
'skip': 'Invalid URL',
}, { }, {
'url': 'https://www.ertflix.gr/series/ser.3448-monogramma', 'url': 'https://www.ertflix.gr/series/ser.3448-monogramma',
'info_dict': { 'info_dict': {
'id': 'ser.3448', 'id': 'ser.3448',
'age_limit': 8, 'age_limit': 8,
'description': 'Η εκπομπή σαράντα ετών που σημάδεψε τον πολιτισμό μας.', 'title': 'Monogramma',
'title': 'Μονόγραμμα', 'description': 'md5:e30cc640e6463da87f210a8ed10b2439',
}, },
'playlist_mincount': 64, 'playlist_mincount': 64,
}, { }, {
@@ -145,28 +144,28 @@ class ERTFlixIE(ERTFlixBaseIE):
'info_dict': { 'info_dict': {
'id': 'ser.3448', 'id': 'ser.3448',
'age_limit': 8, 'age_limit': 8,
'description': 'Η εκπομπή σαράντα ετών που σημάδεψε τον πολιτισμό μας.', 'title': 'Monogramma',
'title': 'Μονόγραμμα', 'description': 'md5:e30cc640e6463da87f210a8ed10b2439',
}, },
'playlist_count': 22, 'playlist_mincount': 66,
}, { }, {
'url': 'https://www.ertflix.gr/series/ser.3448-monogramma?season=1&season=2021%20-%202022', 'url': 'https://www.ertflix.gr/series/ser.3448-monogramma?season=1&season=2021%20-%202022',
'info_dict': { 'info_dict': {
'id': 'ser.3448', 'id': 'ser.3448',
'age_limit': 8, 'age_limit': 8,
'description': 'Η εκπομπή σαράντα ετών που σημάδεψε τον πολιτισμό μας.', 'title': 'Monogramma',
'title': 'Μονόγραμμα', 'description': 'md5:e30cc640e6463da87f210a8ed10b2439',
}, },
'playlist_mincount': 36, 'playlist_mincount': 25,
}, { }, {
'url': 'https://www.ertflix.gr/series/ser.164991-to-diktuo-1?season=1-9', 'url': 'https://www.ertflix.gr/series/ser.164991-to-diktuo-1?season=1-9',
'info_dict': { 'info_dict': {
'id': 'ser.164991', 'id': 'ser.164991',
'age_limit': 8, 'age_limit': 8,
'description': 'Η πρώτη ελληνική εκπομπή με θεματολογία αποκλειστικά γύρω από το ίντερνετ.', 'title': 'The Network',
'title': 'Το δίκτυο', 'description': 'The first Greek show featuring topics exclusively around the internet.',
}, },
'playlist_mincount': 9, 'playlist_mincount': 0,
}, { }, {
'url': 'https://www.ertflix.gr/en/vod/vod.127652-ta-kalytera-mas-chronia-ep1-mia-volta-sto-feggari', 'url': 'https://www.ertflix.gr/en/vod/vod.127652-ta-kalytera-mas-chronia-ep1-mia-volta-sto-feggari',
'only_matching': True, 'only_matching': True,
@@ -282,6 +281,16 @@ class ERTWebtvEmbedIE(InfoExtractor):
'ext': 'mp4', 'ext': 'mp4',
'thumbnail': 'https://program.ert.gr/photos/2022/1/to_diktio_ep09_i_istoria_tou_diadiktiou_stin_Ellada_1021x576.jpg', 'thumbnail': 'https://program.ert.gr/photos/2022/1/to_diktio_ep09_i_istoria_tou_diadiktiou_stin_Ellada_1021x576.jpg',
}, },
'skip': 'Invalid URL',
}]
_WEBPAGE_TESTS = [{
'url': 'https://www.ertnews.gr/video/manolis-goyalles-o-anthropos-piso-apo-ti-diadiktyaki-vasilopita/',
'info_dict': {
'id': '2022/tv/news-themata-ianouarios/20220114-apotis6-gouales-pita.mp4',
'ext': 'mp4',
'title': 'VOD - 2022/tv/news-themata-ianouarios/20220114-apotis6-gouales-pita.mp4',
'thumbnail': r're:https?://www\.ert\.gr/themata/photos/.+\.jpg',
},
}] }]
def _real_extract(self, url): def _real_extract(self, url):
plugins/youtube_download/yt_dlp/extractor/espn.py
+5 -7
View File
@@ -5,7 +5,6 @@ import urllib.parse
from .adobepass import AdobePassIE from .adobepass import AdobePassIE
from .common import InfoExtractor from .common import InfoExtractor
from .once import OnceIE
from ..utils import ( from ..utils import (
determine_ext, determine_ext,
dict_get, dict_get,
@@ -16,7 +15,7 @@ from ..utils import (
) )
class ESPNIE(OnceIE): class ESPNIE(InfoExtractor):
_VALID_URL = r'''(?x) _VALID_URL = r'''(?x)
https?:// https?://
(?: (?:
@@ -131,9 +130,7 @@ class ESPNIE(OnceIE):
return return
format_urls.add(source_url) format_urls.add(source_url)
ext = determine_ext(source_url) ext = determine_ext(source_url)
if OnceIE.suitable(source_url): if ext == 'smil':
formats.extend(self._extract_once_formats(source_url))
elif ext == 'smil':
formats.extend(self._extract_smil_formats( formats.extend(self._extract_smil_formats(
source_url, video_id, fatal=False)) source_url, video_id, fatal=False))
elif ext == 'f4m': elif ext == 'f4m':
@@ -332,6 +329,7 @@ class WatchESPNIE(AdobePassIE):
}] }]
_API_KEY = 'ZXNwbiZicm93c2VyJjEuMC4w.ptUt7QxsteaRruuPmGZFaJByOoqKvDP2a5YkInHrc7c' _API_KEY = 'ZXNwbiZicm93c2VyJjEuMC4w.ptUt7QxsteaRruuPmGZFaJByOoqKvDP2a5YkInHrc7c'
_SOFTWARE_STATEMENT = 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIyZGJmZWM4My03OWE1LTQyNzEtYTVmZC04NTZjYTMxMjRjNjMiLCJuYmYiOjE1NDAyMTI3NjEsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTQwMjEyNzYxfQ.yaK3r4AI2uLVvsyN1GLzqzgzRlxMPtasSaiYYBV0wIstqih5tvjTmeoLmi8Xy9Kp_U7Md-bOffwiyK3srHkpUkhhwXLH2x6RPjmS1tPmhaG7-3LBcHTf2ySPvXhVf7cN4ngldawK4tdtLtsw6rF_JoZE2yaC6XbS2F51nXSFEDDnOQWIHEQRG3aYAj-38P2CLGf7g-Yfhbp5cKXeksHHQ90u3eOO4WH0EAjc9oO47h33U8KMEXxJbvjV5J8Va2G2fQSgLDZ013NBI3kQnE313qgqQh2feQILkyCENpB7g-TVBreAjOaH1fU471htSoGGYepcAXv-UDtpgitDiLy7CQ'
def _call_bamgrid_api(self, path, video_id, payload=None, headers={}): def _call_bamgrid_api(self, path, video_id, payload=None, headers={}):
if 'Authorization' not in headers: if 'Authorization' not in headers:
@@ -408,8 +406,8 @@ class WatchESPNIE(AdobePassIE):
# TV Provider required # TV Provider required
else: else:
resource = self._get_mvpd_resource('ESPN', video_data['name'], video_id, None) resource = self._get_mvpd_resource('espn1', video_data['name'], video_id, None)
auth = self._extract_mvpd_auth(url, video_id, 'ESPN', resource).encode() auth = self._extract_mvpd_auth(url, video_id, 'ESPN', resource, self._SOFTWARE_STATEMENT).encode()
asset = self._download_json( asset = self._download_json(
f'https://watch.auth.api.espn.com/video/auth/media/{video_id}/asset?apikey=uiqlbgzdwuru14v627vdusswb', f'https://watch.auth.api.espn.com/video/auth/media/{video_id}/asset?apikey=uiqlbgzdwuru14v627vdusswb',
plugins/youtube_download/yt_dlp/extractor/facebook.py
+98 -46
View File
@@ -81,13 +81,14 @@ class FacebookIE(InfoExtractor):
'description': 'md5:34675bda53336b1d16400265c2bb9b3b', 'description': 'md5:34675bda53336b1d16400265c2bb9b3b',
'uploader': 'RADIO KICKS FM', 'uploader': 'RADIO KICKS FM',
'upload_date': '20230818', 'upload_date': '20230818',
'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'timestamp': 1692346159, 'timestamp': 1692346159,
'thumbnail': r're:^https?://.*',
'uploader_id': '100063551323670', 'uploader_id': '100063551323670',
'duration': 3133.583, 'duration': 3133.583,
'view_count': int, 'view_count': int,
'concurrent_view_count': 0, 'concurrent_view_count': 0,
}, },
'expected_warnings': ['Cannot parse data'],
}, { }, {
'url': 'https://www.facebook.com/video.php?v=637842556329505&fref=nf', 'url': 'https://www.facebook.com/video.php?v=637842556329505&fref=nf',
'md5': '6a40d33c0eccbb1af76cf0485a052659', 'md5': '6a40d33c0eccbb1af76cf0485a052659',
@@ -106,17 +107,18 @@ class FacebookIE(InfoExtractor):
'info_dict': { 'info_dict': {
'id': '274175099429670', 'id': '274175099429670',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Asif', 'title': '119 reactions · 1.4K shares | Asif Nawab Butt on Reels',
'description': '', 'description': '',
'uploader': 'Asif Nawab Butt', 'uploader': 'Asif Nawab Butt',
'upload_date': '20140506', 'upload_date': '20140506',
'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'timestamp': 1399398998, 'timestamp': 1399398998,
'thumbnail': r're:^https?://.*', 'uploader_id': 'pfbid028xue38TBXRyNbiqBSV2LFs3QK3yopvKjupbqFoL6U9SKbx4p2SMdJjQSBvnjsHGWl',
'uploader_id': 'pfbid05AzrFTXgY37tqwaSgbFTTEpCLBjjEJHkigogwGiRPtKEpAsJYJpzE94H1RxYXWEtl',
'duration': 131.03, 'duration': 131.03,
'concurrent_view_count': int, 'concurrent_view_count': int,
'view_count': int, 'view_count': int,
}, },
'expected_warnings': ['Cannot parse data'],
}, { }, {
'note': 'Video with DASH manifest', 'note': 'Video with DASH manifest',
'url': 'https://www.facebook.com/video.php?v=957955867617029', 'url': 'https://www.facebook.com/video.php?v=957955867617029',
@@ -158,7 +160,7 @@ class FacebookIE(InfoExtractor):
'id': '10153664894881749', 'id': '10153664894881749',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Average time to confirm recent Supreme Court nominees: 67 days Longest it\'s t...', 'title': 'Average time to confirm recent Supreme Court nominees: 67 days Longest it\'s t...',
'thumbnail': r're:^https?://.*', 'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'timestamp': 1456259628, 'timestamp': 1456259628,
'upload_date': '20160223', 'upload_date': '20160223',
'uploader': 'Barack Obama', 'uploader': 'Barack Obama',
@@ -168,7 +170,7 @@ class FacebookIE(InfoExtractor):
# have 1080P, but only up to 720p in swf params # have 1080P, but only up to 720p in swf params
# data.video.story.attachments[].media # data.video.story.attachments[].media
'url': 'https://www.facebook.com/cnn/videos/10155529876156509/', 'url': 'https://www.facebook.com/cnn/videos/10155529876156509/',
'md5': '1659aa21fb3dd1585874f668e81a72c8', 'md5': '70b82ebf5f0e9b91b2a49d3db3563611',
'info_dict': { 'info_dict': {
'id': '10155529876156509', 'id': '10155529876156509',
'ext': 'mp4', 'ext': 'mp4',
@@ -177,7 +179,7 @@ class FacebookIE(InfoExtractor):
'timestamp': 1477818095, 'timestamp': 1477818095,
'upload_date': '20161030', 'upload_date': '20161030',
'uploader': 'CNN', 'uploader': 'CNN',
'thumbnail': r're:^https?://.*', 'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'view_count': int, 'view_count': int,
'uploader_id': '100059479812265', 'uploader_id': '100059479812265',
'concurrent_view_count': int, 'concurrent_view_count': int,
@@ -198,13 +200,11 @@ class FacebookIE(InfoExtractor):
'uploader': 'Yaroslav Korpan', 'uploader': 'Yaroslav Korpan',
'uploader_id': 'pfbid06AScABAWcW91qpiuGrLt99Ef9tvwHoXP6t8KeFYEqkSfreMtfa9nTveh8b2ZEVSWl', 'uploader_id': 'pfbid06AScABAWcW91qpiuGrLt99Ef9tvwHoXP6t8KeFYEqkSfreMtfa9nTveh8b2ZEVSWl',
'concurrent_view_count': int, 'concurrent_view_count': int,
'thumbnail': r're:^https?://.*', 'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'view_count': int, 'view_count': int,
'duration': 11736.446, 'duration': 11736.446,
}, },
'params': { 'skip': 'Invalid URL',
'skip_download': True,
},
}, { }, {
# FIXME: Cannot parse data error # FIXME: Cannot parse data error
'url': 'https://www.facebook.com/LaGuiaDelVaron/posts/1072691702860471', 'url': 'https://www.facebook.com/LaGuiaDelVaron/posts/1072691702860471',
@@ -215,7 +215,7 @@ class FacebookIE(InfoExtractor):
'timestamp': 1477305000, 'timestamp': 1477305000,
'upload_date': '20161024', 'upload_date': '20161024',
'uploader': 'La Guía Del Varón', 'uploader': 'La Guía Del Varón',
'thumbnail': r're:^https?://.*', 'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
}, },
'skip': 'Requires logging in', 'skip': 'Requires logging in',
}, { }, {
@@ -244,9 +244,10 @@ class FacebookIE(InfoExtractor):
'upload_date': '20171124', 'upload_date': '20171124',
'uploader': 'Vickie Gentry', 'uploader': 'Vickie Gentry',
'uploader_id': 'pfbid0FkkycT95ySNNyfCw4Cho6u5G7WbbZEcxT496Hq8rtx1K3LcTCATpR3wnyYhmyGC5l', 'uploader_id': 'pfbid0FkkycT95ySNNyfCw4Cho6u5G7WbbZEcxT496Hq8rtx1K3LcTCATpR3wnyYhmyGC5l',
'thumbnail': r're:^https?://.*', 'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'duration': 148.224, 'duration': 148.224,
}, },
'skip': 'Invalid URL',
}, { }, {
# data.node.comet_sections.content.story.attachments[].styles.attachment.media # data.node.comet_sections.content.story.attachments[].styles.attachment.media
'url': 'https://www.facebook.com/attn/posts/pfbid0j1Czf2gGDVqeQ8KiMLFm3pWN8GxsQmeRrVhimWDzMuKQoR8r4b1knNsejELmUgyhl', 'url': 'https://www.facebook.com/attn/posts/pfbid0j1Czf2gGDVqeQ8KiMLFm3pWN8GxsQmeRrVhimWDzMuKQoR8r4b1knNsejELmUgyhl',
@@ -260,7 +261,7 @@ class FacebookIE(InfoExtractor):
'duration': 132.675, 'duration': 132.675,
'uploader_id': '100064451419378', 'uploader_id': '100064451419378',
'view_count': int, 'view_count': int,
'thumbnail': r're:^https?://.*', 'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'timestamp': 1701975646, 'timestamp': 1701975646,
}, },
}, { }, {
@@ -271,9 +272,9 @@ class FacebookIE(InfoExtractor):
'ext': 'mp4', 'ext': 'mp4',
'title': 'Lela Evans', 'title': 'Lela Evans',
'description': 'Today Makkovik\'s own Pilot Mandy Smith made her inaugural landing on the airstrip in her hometown. What a proud moment as we all cheered and...', 'description': 'Today Makkovik\'s own Pilot Mandy Smith made her inaugural landing on the airstrip in her hometown. What a proud moment as we all cheered and...',
'thumbnail': r're:^https?://.*', 'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'uploader': 'Lela Evans', 'uploader': 'Lela Evans',
'uploader_id': 'pfbid0swT2y7t6TAsZVBvcyeYPdhTMefGaS26mzUwML3vd1ma6ndGZKxsyS4Ssu3jitZLXl', 'uploader_id': 'pfbid02wjMpknobSMnyynK3TNKN4Ww1StcpAKXgowqTyge3bz7LwHZMQ68uiXzzbu7xeryBl',
'upload_date': '20231228', 'upload_date': '20231228',
'timestamp': 1703804085, 'timestamp': 1703804085,
'duration': 394.347, 'duration': 394.347,
@@ -326,28 +327,27 @@ class FacebookIE(InfoExtractor):
'uploader_id': '100066514874195', 'uploader_id': '100066514874195',
'duration': 4524.001, 'duration': 4524.001,
'view_count': int, 'view_count': int,
'thumbnail': r're:^https?://.*', 'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'concurrent_view_count': int, 'concurrent_view_count': int,
}, },
'params': { 'params': {'skip_download': True},
'skip_download': True,
},
}, { }, {
# data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.all_subattachments.nodes[].media # data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.all_subattachments.nodes[].media
'url': 'https://www.facebook.com/100033620354545/videos/106560053808006/', 'url': 'https://www.facebook.com/100033620354545/videos/106560053808006/',
'info_dict': { 'info_dict': {
'id': '106560053808006', 'id': '106560053808006',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Josef', 'title': 'Josef Novak on Reels',
'thumbnail': r're:^https?://.*', 'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'concurrent_view_count': int, 'concurrent_view_count': int,
'uploader_id': 'pfbid02gpfwRM2XvdEJfsERupwQiNmBiDArc38RMRYZnap372q6Vs7MtFTVy72mmFWpJBTKl', 'uploader_id': 'pfbid0cjYJYXpePWqhZ9DgpB6gKXrN2q3obwducdKm4wT7K5nkhbfKg5cneocYbsdaji7fl',
'timestamp': 1549275572, 'timestamp': 1549275572,
'duration': 3.283, 'duration': 3.283,
'uploader': 'Josef Novak', 'uploader': 'Josef Novak',
'description': '', 'description': '',
'upload_date': '20190204', 'upload_date': '20190204',
}, },
'expected_warnings': ['Cannot parse data'],
}, { }, {
# data.video.story.attachments[].media # data.video.story.attachments[].media
'url': 'https://www.facebook.com/watch/?v=647537299265662', 'url': 'https://www.facebook.com/watch/?v=647537299265662',
@@ -406,7 +406,7 @@ class FacebookIE(InfoExtractor):
'ext': 'mp4', 'ext': 'mp4',
'title': 'ANALISI IN CAMPO OSCURO " Coaguli nel sangue dei vaccinati"', 'title': 'ANALISI IN CAMPO OSCURO " Coaguli nel sangue dei vaccinati"',
'description': 'Other event by Comitato Liberi Pensatori on Tuesday, October 18 2022', 'description': 'Other event by Comitato Liberi Pensatori on Tuesday, October 18 2022',
'thumbnail': r're:^https?://.*', 'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'uploader': 'Comitato Liberi Pensatori', 'uploader': 'Comitato Liberi Pensatori',
'uploader_id': '100065709540881', 'uploader_id': '100065709540881',
}, },
@@ -414,6 +414,56 @@ class FacebookIE(InfoExtractor):
'url': 'https://www.facebook.com/groups/1513990329015294/posts/d41d8cd9/2013209885760000/?app=fbl', 'url': 'https://www.facebook.com/groups/1513990329015294/posts/d41d8cd9/2013209885760000/?app=fbl',
'only_matching': True, 'only_matching': True,
}] }]
_WEBPAGE_TESTS = [{
# <iframe> embed
'url': 'http://www.unique-almeria.com/mini-hollywood.html',
'md5': 'cba5d8c5021e9340dcefe925255e2c3e',
'info_dict': {
'id': '1529066599879',
'ext': 'mp4',
'title': 'Facebook video #1529066599879',
},
'expected_warnings': ['unable to extract uploader'],
}, {
# FIXME: Embed detection
# <iframe> embed, plugin video
'url': 'https://www.newsmemory.com/eedition/e-publishing-solutions/2-in-one-app/',
'md5': 'ae97d4a44f8cc9a8b1a4c03b9ed793af',
'info_dict': {
'id': '10155710648695814',
'ext': 'mp4',
'title': 'Download the all new and improved Trinidad Express App',
'concurrent_view_count': int,
'description': 'md5:4806195c99908e4189b45b1c23bd4f89',
'duration': 69.408,
'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'timestamp': 1533919195,
'upload_date': '20180810',
'uploader': 'Trinidad Express Newspapers',
'uploader_id': '100064446413648',
'view_count': int,
},
'expected_warnings': ['Cannot parse data'],
}, {
# API embed
'url': 'https://www.curs.md/ro',
'md5': '090bae53b9bff2be993c896edc2ea205',
'info_dict': {
'id': '334484292523563',
'ext': 'mp4',
'title': 'md5:9abffe1c86cdd967ffa224e1ccc13b90',
'concurrent_view_count': int,
'description': 'md5:0ba98567a61c640f9fabf1882235b33d',
'duration': 8789.891,
'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'timestamp': 1700603114,
'upload_date': '20231121',
'uploader': 'Istoria Moldovei',
'uploader_id': '100063529778592',
'view_count': int,
},
'params': {'extractor_args': {'generic': {'impersonate': ['chrome']}}},
}]
_SUPPORTED_PAGLETS_REGEX = r'(?:pagelet_group_mall|permalink_video_pagelet|hyperfeed_story_id_[0-9a-f]+)' _SUPPORTED_PAGLETS_REGEX = r'(?:pagelet_group_mall|permalink_video_pagelet|hyperfeed_story_id_[0-9a-f]+)'
_api_config = { _api_config = {
'graphURI': '/api/graphql/', 'graphURI': '/api/graphql/',
@@ -898,20 +948,24 @@ class FacebookIE(InfoExtractor):
class FacebookPluginsVideoIE(InfoExtractor): class FacebookPluginsVideoIE(InfoExtractor):
_VALID_URL = r'https?://(?:[\w-]+\.)?facebook\.com/plugins/video\.php\?.*?\bhref=(?P<id>https.+)' _VALID_URL = r'https?://(?:[\w-]+\.)?facebook\.com/plugins/video\.php\?.*?\bhref=(?P<id>https.+)'
_TESTS = [{ _TESTS = [{
'url': 'https://www.facebook.com/plugins/video.php?href=https%3A%2F%2Fwww.facebook.com%2Fgov.sg%2Fvideos%2F10154383743583686%2F&show_text=0&width=560', 'url': 'https://www.facebook.com/plugins/video.php?href=https%3A%2F%2Fwww.facebook.com%2Fgov.sg%2Fvideos%2F10154383743583686%2F&show_text=0&width=560',
'md5': '5954e92cdfe51fe5782ae9bda7058a07', 'md5': 'af83aeae1d595f377c6e47a450828155',
'info_dict': { 'info_dict': {
'id': '10154383743583686', 'id': '10154383743583686',
'ext': 'mp4', 'ext': 'mp4',
# TODO: Fix title, uploader
'title': 'What to do during the haze?', 'title': 'What to do during the haze?',
'uploader': 'Gov.sg', 'concurrent_view_count': int,
'upload_date': '20160826', 'description': 'md5:81839c0979803a014b20798df255ed0b',
'duration': 65.087,
'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'timestamp': 1472184808, 'timestamp': 1472184808,
'upload_date': '20160826',
'uploader': 'gov.sg',
'uploader_id': '100064718678925',
'view_count': int,
}, },
'add_ie': [FacebookIE.ie_key()], 'expected_warnings': ['Cannot parse data'],
}, { }, {
'url': 'https://www.facebook.com/plugins/video.php?href=https%3A%2F%2Fwww.facebook.com%2Fvideo.php%3Fv%3D10204634152394104', 'url': 'https://www.facebook.com/plugins/video.php?href=https%3A%2F%2Fwww.facebook.com%2Fvideo.php%3Fv%3D10204634152394104',
'only_matching': True, 'only_matching': True,
@@ -945,7 +999,7 @@ class FacebookRedirectURLIE(InfoExtractor):
'tags': 'count:11', 'tags': 'count:11',
'duration': 3332, 'duration': 3332,
'live_status': 'not_live', 'live_status': 'not_live',
'thumbnail': 'https://i.ytimg.com/vi/pO8h3EaFRdo/maxresdefault.jpg', 'thumbnail': r're:https?://i\.ytimg\.com/vi/.+',
'channel_url': 'https://www.youtube.com/channel/UCGBpxWJr9FNOcFYA5GkKrMg', 'channel_url': 'https://www.youtube.com/channel/UCGBpxWJr9FNOcFYA5GkKrMg',
'availability': 'public', 'availability': 'public',
'uploader_url': 'http://www.youtube.com/user/brtvofficial', 'uploader_url': 'http://www.youtube.com/user/brtvofficial',
@@ -954,8 +1008,7 @@ class FacebookRedirectURLIE(InfoExtractor):
'view_count': int, 'view_count': int,
'like_count': int, 'like_count': int,
}, },
'add_ie': ['Youtube'], 'skip': 'Youtube video is now private',
'params': {'skip_download': 'Youtube'},
}] }]
def _real_extract(self, url): def _real_extract(self, url):
@@ -968,22 +1021,20 @@ class FacebookRedirectURLIE(InfoExtractor):
class FacebookReelIE(InfoExtractor): class FacebookReelIE(InfoExtractor):
_VALID_URL = r'https?://(?:[\w-]+\.)?facebook\.com/reel/(?P<id>\d+)' _VALID_URL = r'https?://(?:[\w-]+\.)?facebook\.com/reel/(?P<id>\d+)'
IE_NAME = 'facebook:reel' IE_NAME = 'facebook:reel'
_TESTS = [{ _TESTS = [{
'url': 'https://www.facebook.com/reel/1195289147628387', 'url': 'https://www.facebook.com/reel/1195289147628387',
'md5': 'a53256d10fc2105441fe0c4212ed8cea', 'md5': 'aeb0153ecb2eaacdf2dc2bf88f593fef',
'info_dict': { 'info_dict': {
'id': '1195289147628387', 'id': '1195289147628387',
'ext': 'mp4', 'ext': 'mp4',
'title': r're:9\.6K views · 355 reactions .+ Let the “Slapathon” commence!! .+ LL COOL J · Mama Said Knock You Out$', 'title': '9.7K views · 352 reactions | When your trying to help your partner out with an arrest and #FAAFO games begin. Let the “Slapathon” commence!! 👊👋 | Beast Camp Training',
'description': r're:When your trying to help your partner .+ LL COOL J · Mama Said Knock You Out$', 'description': 'md5:5a767dc7e78718667b150a7facc4a34f',
'uploader': 'Beast Camp Training', 'uploader': '9.7K views &#xb7; 352 reactions | When your trying to help your partner out with an arrest and #FAAFO games begin. Let the &#x201c;Slapathon&#x201d; commence!! &#x1f44a;&#x1f44b; | Beast Camp Training',
'uploader_id': '100040874179269', 'uploader_id': '100040874179269',
'duration': 9.579, 'duration': 9.579,
'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'timestamp': 1637502609, 'timestamp': 1637502609,
'upload_date': '20211121', 'upload_date': '20211121',
'thumbnail': r're:^https?://.*',
'like_count': int,
'comment_count': int, 'comment_count': int,
'repost_count': int, 'repost_count': int,
}, },
@@ -998,7 +1049,6 @@ class FacebookReelIE(InfoExtractor):
class FacebookAdsIE(InfoExtractor): class FacebookAdsIE(InfoExtractor):
_VALID_URL = r'https?://(?:[\w-]+\.)?facebook\.com/ads/library/?\?(?:[^#]+&)?id=(?P<id>\d+)' _VALID_URL = r'https?://(?:[\w-]+\.)?facebook\.com/ads/library/?\?(?:[^#]+&)?id=(?P<id>\d+)'
IE_NAME = 'facebook:ads' IE_NAME = 'facebook:ads'
_TESTS = [{ _TESTS = [{
'url': 'https://www.facebook.com/ads/library/?id=899206155126718', 'url': 'https://www.facebook.com/ads/library/?id=899206155126718',
'info_dict': { 'info_dict': {
@@ -1008,12 +1058,13 @@ class FacebookAdsIE(InfoExtractor):
'description': 'md5:0822724069e3aca97cbed5dabbab282e', 'description': 'md5:0822724069e3aca97cbed5dabbab282e',
'uploader': 'Kandao', 'uploader': 'Kandao',
'uploader_id': '774114102743284', 'uploader_id': '774114102743284',
'uploader_url': r're:^https?://.*', 'uploader_url': 'https://facebook.com/KandaoVR',
'timestamp': 1702548330, 'timestamp': 1702548330,
'thumbnail': r're:^https?://.*', 'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'upload_date': '20231214', 'upload_date': '20231214',
'like_count': int, 'like_count': int,
}, },
'skip': 'Invalid URL',
}, { }, {
# key 'watermarked_video_sd_url' missing # key 'watermarked_video_sd_url' missing
'url': 'https://www.facebook.com/ads/library/?id=501152689226254', 'url': 'https://www.facebook.com/ads/library/?id=501152689226254',
@@ -1024,9 +1075,9 @@ class FacebookAdsIE(InfoExtractor):
'description': 'md5:02a446ace7ff8c3c37a2892922492490', 'description': 'md5:02a446ace7ff8c3c37a2892922492490',
'uploader': 'mat.nawrocki', 'uploader': 'mat.nawrocki',
'uploader_id': '148586968341456', 'uploader_id': '148586968341456',
'uploader_url': r're:^https?://.*', 'uploader_url': 'https://www.instagram.com/_u/mat.nawrocki',
'thumbnail': r're:https?://scontent\.fitm\d-1\.fna\.fbcdn\.net/.+',
'timestamp': 1723452305, 'timestamp': 1723452305,
'thumbnail': r're:^https?://.*',
'upload_date': '20240812', 'upload_date': '20240812',
'like_count': int, 'like_count': int,
}, },
@@ -1037,12 +1088,13 @@ class FacebookAdsIE(InfoExtractor):
'title': 'Jusqu\u2019\u00e0 -25% sur une s\u00e9lection de vins p\u00e9tillants italiens ', 'title': 'Jusqu\u2019\u00e0 -25% sur une s\u00e9lection de vins p\u00e9tillants italiens ',
'uploader': 'Eataly Paris Marais', 'uploader': 'Eataly Paris Marais',
'uploader_id': '2086668958314152', 'uploader_id': '2086668958314152',
'uploader_url': r're:^https?://.*', 'uploader_url': 'https://facebook.com/EatalyParisMarais',
'timestamp': 1703571529, 'timestamp': 1703571529,
'upload_date': '20231226', 'upload_date': '20231226',
'like_count': int, 'like_count': int,
}, },
'playlist_count': 3, 'playlist_count': 3,
'skip': 'Invalid URL',
}, { }, {
'url': 'https://es-la.facebook.com/ads/library/?id=901230958115569', 'url': 'https://es-la.facebook.com/ads/library/?id=901230958115569',
'only_matching': True, 'only_matching': True,
plugins/youtube_download/yt_dlp/extractor/faulio.py
+237
View File
@@ -0,0 +1,237 @@
import re
import urllib.parse
from .common import InfoExtractor
from ..utils import int_or_none, js_to_json, url_or_none
from ..utils.traversal import traverse_obj
class FaulioBaseIE(InfoExtractor):
_DOMAINS = (
'aloula.sba.sa',
'bahry.com',
'maraya.sba.net.ae',
'sat7plus.org',
)
_LANGUAGES = ('ar', 'en', 'fa')
_BASE_URL_RE = fr'https?://(?:{"|".join(map(re.escape, _DOMAINS))})/(?:(?:{"|".join(_LANGUAGES)})/)?'
def _get_headers(self, url):
parsed_url = urllib.parse.urlparse(url)
return {
'Referer': url,
'Origin': f'{parsed_url.scheme}://{parsed_url.hostname}',
}
def _get_api_base(self, url, video_id):
webpage = self._download_webpage(url, video_id)
config_data = self._search_json(
r'window\.__NUXT__\.config=', webpage, 'config', video_id, transform_source=js_to_json)
return config_data['public']['TRANSLATIONS_API_URL']
class FaulioIE(FaulioBaseIE):
_VALID_URL = fr'{FaulioBaseIE._BASE_URL_RE}(?:episode|media)/(?P<id>[a-zA-Z0-9-]+)'
_TESTS = [{
'url': 'https://aloula.sba.sa/en/episode/29102',
'info_dict': {
'id': 'aloula.faulio.com_29102',
'ext': 'mp4',
'display_id': 'هذا-مكانك-03-004-v-29102',
'title': 'الحلقة 4',
'episode': 'الحلقة 4',
'description': '',
'series': 'هذا مكانك',
'season': 'Season 3',
'season_number': 3,
'episode_number': 4,
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 4855,
'age_limit': 3,
},
}, {
'url': 'https://bahry.com/en/media/1191',
'info_dict': {
'id': 'bahry.faulio.com_1191',
'ext': 'mp4',
'display_id': 'Episode-4-1191',
'title': 'Episode 4',
'episode': 'Episode 4',
'description': '',
'series': 'Wild Water',
'season': 'Season 1',
'season_number': 1,
'episode_number': 4,
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 1653,
'age_limit': 0,
},
}, {
'url': 'https://maraya.sba.net.ae/episode/127735',
'info_dict': {
'id': 'maraya.faulio.com_127735',
'ext': 'mp4',
'display_id': 'عبدالله-الهاجري---عبدالرحمن-المطروشي-127735',
'title': 'عبدالله الهاجري - عبدالرحمن المطروشي',
'episode': 'عبدالله الهاجري - عبدالرحمن المطروشي',
'description': 'md5:53de01face66d3d6303221e5a49388a0',
'series': 'أبناؤنا في الخارج',
'season': 'Season 3',
'season_number': 3,
'episode_number': 7,
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 1316,
'age_limit': 0,
},
}, {
'url': 'https://sat7plus.org/episode/18165',
'info_dict': {
'id': 'sat7.faulio.com_18165',
'ext': 'mp4',
'display_id': 'ep-13-ADHD-18165',
'title': 'ADHD and creativity',
'episode': 'ADHD and creativity',
'description': '',
'series': 'ADHD Podcast',
'season': 'Season 1',
'season_number': 1,
'episode_number': 13,
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 2492,
'age_limit': 0,
},
}, {
'url': 'https://aloula.sba.sa/en/episode/0',
'only_matching': True,
}]
def _real_extract(self, url):
video_id = self._match_id(url)
api_base = self._get_api_base(url, video_id)
video_info = self._download_json(f'{api_base}/video/{video_id}', video_id, fatal=False)
player_info = self._download_json(f'{api_base}/video/{video_id}/player', video_id)
headers = self._get_headers(url)
formats = []
subtitles = {}
if hls_url := traverse_obj(player_info, ('settings', 'protocols', 'hls', {url_or_none})):
fmts, subs = self._extract_m3u8_formats_and_subtitles(
hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False, headers=headers)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
if mpd_url := traverse_obj(player_info, ('settings', 'protocols', 'dash', {url_or_none})):
fmts, subs = self._extract_mpd_formats_and_subtitles(
mpd_url, video_id, mpd_id='dash', fatal=False, headers=headers)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
return {
'id': f'{urllib.parse.urlparse(api_base).hostname}_{video_id}',
**traverse_obj(traverse_obj(video_info, ('blocks', 0)), {
'display_id': ('slug', {str}),
'title': ('title', {str}),
'episode': ('title', {str}),
'description': ('description', {str}),
'series': ('program_title', {str}),
'season_number': ('season_number', {int_or_none}),
'episode_number': ('episode', {int_or_none}),
'thumbnail': ('image', {url_or_none}),
'duration': ('duration', 'total', {int_or_none}),
'age_limit': ('age_rating', {int_or_none}),
}),
'formats': formats,
'subtitles': subtitles,
'http_headers': headers,
}
class FaulioLiveIE(FaulioBaseIE):
_VALID_URL = fr'{FaulioBaseIE._BASE_URL_RE}live/(?P<id>[a-zA-Z0-9-]+)'
_TESTS = [{
'url': 'https://aloula.sba.sa/live/saudiatv',
'info_dict': {
'id': 'aloula.faulio.com_saudiatv',
'title': str,
'description': str,
'ext': 'mp4',
'live_status': 'is_live',
},
'params': {
'skip_download': 'Livestream',
},
}, {
'url': 'https://bahry.com/live/1',
'info_dict': {
'id': 'bahry.faulio.com_1',
'title': str,
'description': str,
'ext': 'mp4',
'live_status': 'is_live',
},
'params': {
'skip_download': 'Livestream',
},
}, {
'url': 'https://maraya.sba.net.ae/live/1',
'info_dict': {
'id': 'maraya.faulio.com_1',
'title': str,
'description': str,
'ext': 'mp4',
'live_status': 'is_live',
},
'params': {
'skip_download': 'Livestream',
},
}, {
'url': 'https://sat7plus.org/live/pars',
'info_dict': {
'id': 'sat7.faulio.com_pars',
'title': str,
'description': str,
'ext': 'mp4',
'live_status': 'is_live',
},
'params': {
'skip_download': 'Livestream',
},
}, {
'url': 'https://sat7plus.org/fa/live/arabic',
'only_matching': True,
}]
def _real_extract(self, url):
video_id = self._match_id(url)
api_base = self._get_api_base(url, video_id)
channel = traverse_obj(
self._download_json(f'{api_base}/channels', video_id),
(lambda k, v: v['url'] == video_id, any))
headers = self._get_headers(url)
formats = []
subtitles = {}
if hls_url := traverse_obj(channel, ('streams', 'hls', {url_or_none})):
fmts, subs = self._extract_m3u8_formats_and_subtitles(
hls_url, video_id, 'mp4', m3u8_id='hls', live=True, fatal=False, headers=headers)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
if mpd_url := traverse_obj(channel, ('streams', 'mpd', {url_or_none})):
fmts, subs = self._extract_mpd_formats_and_subtitles(
mpd_url, video_id, mpd_id='dash', fatal=False, headers=headers)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
return {
'id': f'{urllib.parse.urlparse(api_base).hostname}_{video_id}',
**traverse_obj(channel, {
'title': ('title', {str}),
'description': ('description', {str}),
}),
'formats': formats,
'subtitles': subtitles,
'http_headers': headers,
'is_live': True,
}
plugins/youtube_download/yt_dlp/extractor/fc2.py
+17 -2
View File
@@ -22,8 +22,23 @@ class FC2IE(InfoExtractor):
'md5': 'a6ebe8ebe0396518689d963774a54eb7', 'md5': 'a6ebe8ebe0396518689d963774a54eb7',
'info_dict': { 'info_dict': {
'id': '20121103kUan1KHs', 'id': '20121103kUan1KHs',
'ext': 'flv',
'title': 'Boxing again with Puff', 'title': 'Boxing again with Puff',
'ext': 'mp4',
'thumbnail': r're:https?://.+\.jpe?g',
},
'params': {
'skip_download': 'm3u8',
},
}, {
# Direct video url
'url': 'https://video.fc2.com/content/20121209FP73fxDx',
'md5': '066bdb9b3a56a97f49cbf0d0b8a75a1f',
'info_dict': {
'id': '20121209FP73fxDx',
'title': 'Farewelling The Wiggles Live in Sydney Dec 8 2012',
'ext': 'mp4',
'thumbnail': r're:https?://.+\.jpe?g',
'description': 'Saying goodbye to the Wiggles at their Celebration Concert in Sydney, and what a concert that was!',
}, },
}, { }, {
'url': 'http://video.fc2.com/en/content/20150125cEva0hDn/', 'url': 'http://video.fc2.com/en/content/20150125cEva0hDn/',
@@ -104,7 +119,7 @@ class FC2IE(InfoExtractor):
'title': title, 'title': title,
'url': vid_url, 'url': vid_url,
'ext': 'mp4', 'ext': 'mp4',
'protocol': 'm3u8_native', 'protocol': 'm3u8_native' if vidplaylist.get('type') == 2 else 'https',
'description': description, 'description': description,
'thumbnail': thumbnail, 'thumbnail': thumbnail,
} }
plugins/youtube_download/yt_dlp/extractor/fifa.py
+2 -2
View File
@@ -7,7 +7,7 @@ from ..utils import (
class FifaIE(InfoExtractor): class FifaIE(InfoExtractor):
_VALID_URL = r'https?://www\.fifa\.com/fifaplus/(?P<locale>\w{2})/watch/([^#?]+/)?(?P<id>\w+)' _VALID_URL = r'https?://www\.fifa\.com/fifaplus/\w{2}/watch/([^#?]+/)?(?P<id>\w+)'
_TESTS = [{ _TESTS = [{
'url': 'https://www.fifa.com/fifaplus/en/watch/7on10qPcnyLajDDU3ntg6y', 'url': 'https://www.fifa.com/fifaplus/en/watch/7on10qPcnyLajDDU3ntg6y',
'info_dict': { 'info_dict': {
@@ -51,7 +51,7 @@ class FifaIE(InfoExtractor):
}] }]
def _real_extract(self, url): def _real_extract(self, url):
video_id, locale = self._match_valid_url(url).group('id', 'locale') video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id) webpage = self._download_webpage(url, video_id)
preconnect_link = self._search_regex( preconnect_link = self._search_regex(
plugins/youtube_download/yt_dlp/extractor/firsttv.py
+78 -99
View File
@@ -2,11 +2,15 @@ import urllib.parse
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import (
determine_ext,
int_or_none, int_or_none,
qualities, join_nonempty,
mimetype2ext,
parse_qs,
unified_strdate, unified_strdate,
url_or_none, url_or_none,
) )
from ..utils.traversal import traverse_obj
class FirstTVIE(InfoExtractor): class FirstTVIE(InfoExtractor):
@@ -15,40 +19,51 @@ class FirstTVIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?(?:sport)?1tv\.ru/(?:[^/?#]+/)+(?P<id>[^/?#]+)' _VALID_URL = r'https?://(?:www\.)?(?:sport)?1tv\.ru/(?:[^/?#]+/)+(?P<id>[^/?#]+)'
_TESTS = [{ _TESTS = [{
# single format # single format; has item.id
'url': 'http://www.1tv.ru/shows/naedine-so-vsemi/vypuski/gost-lyudmila-senchina-naedine-so-vsemi-vypusk-ot-12-02-2015', 'url': 'https://www.1tv.ru/shows/naedine-so-vsemi/vypuski/gost-lyudmila-senchina-naedine-so-vsemi-vypusk-ot-12-02-2015',
'md5': 'a1b6b60d530ebcf8daacf4565762bbaf', 'md5': '8011ae8e88ff4150107ab9c5a8f5b659',
'info_dict': { 'info_dict': {
'id': '40049', 'id': '40049',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Гость Людмила Сенчина. Наедине со всеми. Выпуск от 12.02.2015', 'title': 'Гость Людмила Сенчина. Наедине со всеми. Выпуск от 12.02.2015',
'thumbnail': r're:^https?://.*\.(?:jpg|JPG)$', 'thumbnail': r're:https?://.+/.+\.jpg',
'upload_date': '20150212', 'upload_date': '20150212',
'duration': 2694, 'duration': 2694,
}, },
'params': {'skip_download': 'm3u8'},
}, { }, {
# multiple formats # multiple formats; has item.id
'url': 'http://www.1tv.ru/shows/dobroe-utro/pro-zdorove/vesennyaya-allergiya-dobroe-utro-fragment-vypuska-ot-07042016', 'url': 'https://www.1tv.ru/shows/dobroe-utro/pro-zdorove/vesennyaya-allergiya-dobroe-utro-fragment-vypuska-ot-07042016',
'info_dict': { 'info_dict': {
'id': '364746', 'id': '364746',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Весенняя аллергия. Доброе утро. Фрагмент выпуска от 07.04.2016', 'title': 'Весенняя аллергия. Доброе утро. Фрагмент выпуска от 07.04.2016',
'thumbnail': r're:^https?://.*\.(?:jpg|JPG)$', 'thumbnail': r're:https?://.+/.+\.jpg',
'upload_date': '20160407', 'upload_date': '20160407',
'duration': 179, 'duration': 179,
'formats': 'mincount:3', 'formats': 'mincount:3',
}, },
'params': { 'params': {'skip_download': 'm3u8'},
'skip_download': True,
},
}, { }, {
'url': 'http://www.1tv.ru/news/issue/2016-12-01/14:00', 'url': 'https://www.1tv.ru/news/issue/2016-12-01/14:00',
'info_dict': { 'info_dict': {
'id': '14:00', 'id': '14:00',
'title': 'Выпуск новостей в 14:00 1 декабря 2016 года. Новости. Первый канал', 'title': 'Выпуск программы «Время» в 20:00 1 декабря 2016 года. Новости. Первый канал',
'description': 'md5:2e921b948f8c1ff93901da78ebdb1dfd', 'thumbnail': 'https://static.1tv.ru/uploads/photo/image/8/big/338448_big_8fc7eb236f.jpg',
}, },
'playlist_count': 13, 'playlist_count': 13,
}, {
# has timestamp; has item.uid but not item.id
'url': 'https://www.1tv.ru/shows/segodnya-vecherom/vypuski/avtory-odnogo-hita-segodnya-vecherom-vypusk-ot-03-05-2025',
'info_dict': {
'id': '270411',
'ext': 'mp4',
'title': 'Авторы одного хита. Сегодня вечером. Выпуск от 03.05.2025',
'thumbnail': r're:https?://.+/.+\.jpg',
'timestamp': 1746286020,
'upload_date': '20250503',
},
'params': {'skip_download': 'm3u8'},
}, { }, {
'url': 'http://www.1tv.ru/shows/tochvtoch-supersezon/vystupleniya/evgeniy-dyatlov-vladimir-vysockiy-koni-priveredlivye-toch-v-toch-supersezon-fragment-vypuska-ot-06-11-2016', 'url': 'http://www.1tv.ru/shows/tochvtoch-supersezon/vystupleniya/evgeniy-dyatlov-vladimir-vysockiy-koni-priveredlivye-toch-v-toch-supersezon-fragment-vypuska-ot-06-11-2016',
'only_matching': True, 'only_matching': True,
@@ -57,96 +72,60 @@ class FirstTVIE(InfoExtractor):
'only_matching': True, 'only_matching': True,
}] }]
def _entries(self, items):
for item in items:
video_id = str(item.get('id') or item['uid'])
formats, subtitles = [], {}
for f in traverse_obj(item, ('sources', lambda _, v: url_or_none(v['src']))):
src = f['src']
ext = mimetype2ext(f.get('type'), default=determine_ext(src))
if ext == 'm3u8':
fmts, subs = self._extract_m3u8_formats_and_subtitles(
src, video_id, 'mp4', m3u8_id='hls', fatal=False)
elif ext == 'mpd':
fmts, subs = self._extract_mpd_formats_and_subtitles(
src, video_id, mpd_id='dash', fatal=False)
else:
tbr = self._search_regex(fr'_(\d{{3,}})\.{ext}', src, 'tbr', default=None)
formats.append({
'url': src,
'ext': ext,
'format_id': join_nonempty('http', ext, tbr),
'tbr': int_or_none(tbr),
# quality metadata of http formats may be incorrect
'quality': -10,
})
continue
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
yield {
**traverse_obj(item, {
'title': ('title', {str}),
'thumbnail': ('poster', {url_or_none}),
'timestamp': ('dvr_begin_at', {int_or_none}),
'upload_date': ('date_air', {unified_strdate}),
'duration': ('duration', {int_or_none}),
}),
'id': video_id,
'formats': formats,
'subtitles': subtitles,
}
def _real_extract(self, url): def _real_extract(self, url):
display_id = self._match_id(url) display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id) webpage = self._download_webpage(url, display_id)
playlist_url = urllib.parse.urljoin(url, self._search_regex( playlist_url = urllib.parse.urljoin(url, self._html_search_regex(
r'data-playlist-url=(["\'])(?P<url>(?:(?!\1).)+)\1', r'data-playlist-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
webpage, 'playlist url', group='url')) webpage, 'playlist url', group='url'))
parsed_url = urllib.parse.urlparse(playlist_url) item_ids = traverse_obj(parse_qs(playlist_url), 'video_id', 'videos_ids[]', 'news_ids[]')
qs = urllib.parse.parse_qs(parsed_url.query) items = traverse_obj(
item_ids = qs.get('videos_ids[]') or qs.get('news_ids[]') self._download_json(playlist_url, display_id),
lambda _, v: v['uid'] and (str(v['uid']) in item_ids if item_ids else True))
items = self._download_json(playlist_url, display_id) return self.playlist_result(
self._entries(items), display_id, self._og_search_title(webpage, default=None),
if item_ids: thumbnail=self._og_search_thumbnail(webpage, default=None))
items = [
item for item in items
if item.get('uid') and str(item['uid']) in item_ids]
else:
items = [items[0]]
entries = []
QUALITIES = ('ld', 'sd', 'hd')
for item in items:
title = item['title']
quality = qualities(QUALITIES)
formats = []
path = None
for f in item.get('mbr', []):
src = url_or_none(f.get('src'))
if not src:
continue
tbr = int_or_none(self._search_regex(
r'_(\d{3,})\.mp4', src, 'tbr', default=None))
if not path:
path = self._search_regex(
r'//[^/]+/(.+?)_\d+\.mp4', src,
'm3u8 path', default=None)
formats.append({
'url': src,
'format_id': f.get('name'),
'tbr': tbr,
'source_preference': quality(f.get('name')),
# quality metadata of http formats may be incorrect
'preference': -10,
})
# m3u8 URL format is reverse engineered from [1] (search for
# master.m3u8). dashEdges (that is currently balancer-vod.1tv.ru)
# is taken from [2].
# 1. http://static.1tv.ru/player/eump1tv-current/eump-1tv.all.min.js?rnd=9097422834:formatted
# 2. http://static.1tv.ru/player/eump1tv-config/config-main.js?rnd=9097422834
if not path and len(formats) == 1:
path = self._search_regex(
r'//[^/]+/(.+?$)', formats[0]['url'],
'm3u8 path', default=None)
if path:
if len(formats) == 1:
m3u8_path = ','
else:
tbrs = [str(t) for t in sorted(f['tbr'] for f in formats)]
m3u8_path = '_,{},{}'.format(','.join(tbrs), '.mp4')
formats.extend(self._extract_m3u8_formats(
f'http://balancer-vod.1tv.ru/{path}{m3u8_path}.urlset/master.m3u8',
display_id, 'mp4',
entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
thumbnail = item.get('poster') or self._og_search_thumbnail(webpage)
duration = int_or_none(item.get('duration') or self._html_search_meta(
'video:duration', webpage, 'video duration', fatal=False))
upload_date = unified_strdate(self._html_search_meta(
'ya:ovs:upload_date', webpage, 'upload date', default=None))
entries.append({
'id': str(item.get('id') or item['uid']),
'thumbnail': thumbnail,
'title': title,
'upload_date': upload_date,
'duration': int_or_none(duration),
'formats': formats,
})
title = self._html_search_regex(
(r'<div class="tv_translation">\s*<h1><a href="[^"]+">([^<]*)</a>',
r"'title'\s*:\s*'([^']+)'"),
webpage, 'title', default=None) or self._og_search_title(
webpage, default=None)
description = self._html_search_regex(
r'<div class="descr">\s*<div>&nbsp;</div>\s*<p>([^<]*)</p></div>',
webpage, 'description', default=None) or self._html_search_meta(
'description', webpage, 'description', default=None)
return self.playlist_result(entries, display_id, title, description)
plugins/youtube_download/yt_dlp/extractor/flextv.py
+22 -7
View File
@@ -3,15 +3,19 @@ from ..networking.exceptions import HTTPError
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
UserNotLive, UserNotLive,
int_or_none,
join_nonempty,
parse_iso8601, parse_iso8601,
str_or_none, str_or_none,
traverse_obj,
url_or_none, url_or_none,
) )
from ..utils.traversal import traverse_obj
class FlexTVIE(InfoExtractor): class FlexTVIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?flextv\.co\.kr/channels/(?P<id>\d+)/live' IE_NAME = 'ttinglive'
IE_DESC = '띵라이브 (formerly FlexTV)'
_VALID_URL = r'https?://(?:www\.)?(?:ttinglive\.com|flextv\.co\.kr)/channels/(?P<id>\d+)/live'
_TESTS = [{ _TESTS = [{
'url': 'https://www.flextv.co.kr/channels/231638/live', 'url': 'https://www.flextv.co.kr/channels/231638/live',
'info_dict': { 'info_dict': {
@@ -36,21 +40,32 @@ class FlexTVIE(InfoExtractor):
try: try:
stream_data = self._download_json( stream_data = self._download_json(
f'https://api.flextv.co.kr/api/channels/{channel_id}/stream', f'https://api.ttinglive.com/api/channels/{channel_id}/stream',
channel_id, query={'option': 'all'}) channel_id, query={'option': 'all'})
except ExtractorError as e: except ExtractorError as e:
if isinstance(e.cause, HTTPError) and e.cause.status == 400: if isinstance(e.cause, HTTPError) and e.cause.status == 400:
raise UserNotLive(video_id=channel_id) raise UserNotLive(video_id=channel_id)
raise raise
playlist_url = stream_data['sources'][0]['url'] formats = []
formats, subtitles = self._extract_m3u8_formats_and_subtitles( for stream in traverse_obj(stream_data, ('sources', ..., {dict})):
playlist_url, channel_id, 'mp4') if stream.get('format') == 'ivs' and url_or_none(stream.get('url')):
formats.extend(self._extract_m3u8_formats(
stream['url'], channel_id, 'mp4', live=True, fatal=False, m3u8_id='ivs'))
for format_type in ['hls', 'flv']:
for data in traverse_obj(stream, (
'urlDetail', format_type, 'resolution', lambda _, v: url_or_none(v['url']))):
formats.append({
'format_id': join_nonempty(format_type, data.get('suffixName'), delim=''),
'url': data['url'],
'height': int_or_none(data.get('resolution')),
'ext': 'mp4' if format_type == 'hls' else 'flv',
'protocol': 'm3u8_native' if format_type == 'hls' else 'http',
})
return { return {
'id': channel_id, 'id': channel_id,
'formats': formats, 'formats': formats,
'subtitles': subtitles,
'is_live': True, 'is_live': True,
**traverse_obj(stream_data, { **traverse_obj(stream_data, {
'title': ('stream', 'title', {str}), 'title': ('stream', 'title', {str}),
plugins/youtube_download/yt_dlp/extractor/floatplane.py
+134 -98
View File
@@ -17,8 +17,140 @@ from ..utils import (
from ..utils.traversal import traverse_obj from ..utils.traversal import traverse_obj
class FloatplaneIE(InfoExtractor): class FloatplaneBaseIE(InfoExtractor):
def _real_extract(self, url):
post_id = self._match_id(url)
post_data = self._download_json(
f'{self._BASE_URL}/api/v3/content/post', post_id, query={'id': post_id},
note='Downloading post data', errnote='Unable to download post data',
impersonate=self._IMPERSONATE_TARGET)
if not any(traverse_obj(post_data, ('metadata', ('hasVideo', 'hasAudio')))):
raise ExtractorError('Post does not contain a video or audio track', expected=True)
uploader_url = format_field(
post_data, [('creator', 'urlname')], f'{self._BASE_URL}/channel/%s/home') or None
common_info = {
'uploader_url': uploader_url,
'channel_url': urljoin(f'{uploader_url}/', traverse_obj(post_data, ('channel', 'urlname'))),
'availability': self._availability(needs_subscription=True),
**traverse_obj(post_data, {
'uploader': ('creator', 'title', {str}),
'uploader_id': ('creator', 'id', {str}),
'channel': ('channel', 'title', {str}),
'channel_id': ('channel', 'id', {str}),
'release_timestamp': ('releaseDate', {parse_iso8601}),
}),
}
items = []
for media in traverse_obj(post_data, (('videoAttachments', 'audioAttachments'), ...)):
media_id = media['id']
media_typ = media.get('type') or 'video'
metadata = self._download_json(
f'{self._BASE_URL}/api/v3/content/{media_typ}', media_id, query={'id': media_id},
note=f'Downloading {media_typ} metadata', impersonate=self._IMPERSONATE_TARGET)
stream = self._download_json(
f'{self._BASE_URL}/api/v2/cdn/delivery', media_id, query={
'type': 'vod' if media_typ == 'video' else 'aod',
'guid': metadata['guid'],
}, note=f'Downloading {media_typ} stream data',
impersonate=self._IMPERSONATE_TARGET)
path_template = traverse_obj(stream, ('resource', 'uri', {str}))
def format_path(params):
path = path_template
for i, val in (params or {}).items():
path = path.replace(f'{{qualityLevelParams.{i}}}', val)
return path
formats = []
for quality in traverse_obj(stream, ('resource', 'data', 'qualityLevels', ...)):
url = urljoin(stream['cdn'], format_path(traverse_obj(
stream, ('resource', 'data', 'qualityLevelParams', quality['name'], {dict}))))
format_id = traverse_obj(quality, ('name', {str}))
hls_aes = {}
m3u8_data = None
# If we need impersonation for the API, then we need it for HLS keys too: extract in advance
if self._IMPERSONATE_TARGET is not None:
m3u8_data = self._download_webpage(
url, media_id, fatal=False, impersonate=self._IMPERSONATE_TARGET, headers=self._HEADERS,
note=join_nonempty('Downloading', format_id, 'm3u8 information', delim=' '),
errnote=join_nonempty('Failed to download', format_id, 'm3u8 information', delim=' '))
if not m3u8_data:
continue
key_url = self._search_regex(
r'#EXT-X-KEY:METHOD=AES-128,URI="(https?://[^"]+)"',
m3u8_data, 'HLS AES key URI', default=None)
if key_url:
urlh = self._request_webpage(
key_url, media_id, fatal=False, impersonate=self._IMPERSONATE_TARGET, headers=self._HEADERS,
note=join_nonempty('Downloading', format_id, 'HLS AES key', delim=' '),
errnote=join_nonempty('Failed to download', format_id, 'HLS AES key', delim=' '))
if urlh:
hls_aes['key'] = urlh.read().hex()
formats.append({
**traverse_obj(quality, {
'format_note': ('label', {str}),
'width': ('width', {int}),
'height': ('height', {int}),
}),
**parse_codecs(quality.get('codecs')),
'url': url,
'ext': determine_ext(url.partition('/chunk.m3u8')[0], 'mp4'),
'format_id': format_id,
'hls_media_playlist_data': m3u8_data,
'hls_aes': hls_aes or None,
})
items.append({
**common_info,
'id': media_id,
**traverse_obj(metadata, {
'title': ('title', {str}),
'duration': ('duration', {int_or_none}),
'thumbnail': ('thumbnail', 'path', {url_or_none}),
}),
'formats': formats,
})
post_info = {
**common_info,
'id': post_id,
'display_id': post_id,
**traverse_obj(post_data, {
'title': ('title', {str}),
'description': ('text', {clean_html}),
'like_count': ('likes', {int_or_none}),
'dislike_count': ('dislikes', {int_or_none}),
'comment_count': ('comments', {int_or_none}),
'thumbnail': ('thumbnail', 'path', {url_or_none}),
}),
'http_headers': self._HEADERS,
}
if len(items) > 1:
return self.playlist_result(items, **post_info)
post_info.update(items[0])
return post_info
class FloatplaneIE(FloatplaneBaseIE):
_VALID_URL = r'https?://(?:(?:www|beta)\.)?floatplane\.com/post/(?P<id>\w+)' _VALID_URL = r'https?://(?:(?:www|beta)\.)?floatplane\.com/post/(?P<id>\w+)'
_BASE_URL = 'https://www.floatplane.com'
_IMPERSONATE_TARGET = None
_HEADERS = {
'Origin': _BASE_URL,
'Referer': f'{_BASE_URL}/',
}
_TESTS = [{ _TESTS = [{
'url': 'https://www.floatplane.com/post/2Yf3UedF7C', 'url': 'https://www.floatplane.com/post/2Yf3UedF7C',
'info_dict': { 'info_dict': {
@@ -170,105 +302,9 @@ class FloatplaneIE(InfoExtractor):
}] }]
def _real_initialize(self): def _real_initialize(self):
if not self._get_cookies('https://www.floatplane.com').get('sails.sid'): if not self._get_cookies(self._BASE_URL).get('sails.sid'):
self.raise_login_required() self.raise_login_required()
def _real_extract(self, url):
post_id = self._match_id(url)
post_data = self._download_json(
'https://www.floatplane.com/api/v3/content/post', post_id, query={'id': post_id},
note='Downloading post data', errnote='Unable to download post data')
if not any(traverse_obj(post_data, ('metadata', ('hasVideo', 'hasAudio')))):
raise ExtractorError('Post does not contain a video or audio track', expected=True)
uploader_url = format_field(
post_data, [('creator', 'urlname')], 'https://www.floatplane.com/channel/%s/home') or None
common_info = {
'uploader_url': uploader_url,
'channel_url': urljoin(f'{uploader_url}/', traverse_obj(post_data, ('channel', 'urlname'))),
'availability': self._availability(needs_subscription=True),
**traverse_obj(post_data, {
'uploader': ('creator', 'title', {str}),
'uploader_id': ('creator', 'id', {str}),
'channel': ('channel', 'title', {str}),
'channel_id': ('channel', 'id', {str}),
'release_timestamp': ('releaseDate', {parse_iso8601}),
}),
}
items = []
for media in traverse_obj(post_data, (('videoAttachments', 'audioAttachments'), ...)):
media_id = media['id']
media_typ = media.get('type') or 'video'
metadata = self._download_json(
f'https://www.floatplane.com/api/v3/content/{media_typ}', media_id, query={'id': media_id},
note=f'Downloading {media_typ} metadata')
stream = self._download_json(
'https://www.floatplane.com/api/v2/cdn/delivery', media_id, query={
'type': 'vod' if media_typ == 'video' else 'aod',
'guid': metadata['guid'],
}, note=f'Downloading {media_typ} stream data')
path_template = traverse_obj(stream, ('resource', 'uri', {str}))
def format_path(params):
path = path_template
for i, val in (params or {}).items():
path = path.replace(f'{{qualityLevelParams.{i}}}', val)
return path
formats = []
for quality in traverse_obj(stream, ('resource', 'data', 'qualityLevels', ...)):
url = urljoin(stream['cdn'], format_path(traverse_obj(
stream, ('resource', 'data', 'qualityLevelParams', quality['name'], {dict}))))
formats.append({
**traverse_obj(quality, {
'format_id': ('name', {str}),
'format_note': ('label', {str}),
'width': ('width', {int}),
'height': ('height', {int}),
}),
**parse_codecs(quality.get('codecs')),
'url': url,
'ext': determine_ext(url.partition('/chunk.m3u8')[0], 'mp4'),
})
items.append({
**common_info,
'id': media_id,
**traverse_obj(metadata, {
'title': ('title', {str}),
'duration': ('duration', {int_or_none}),
'thumbnail': ('thumbnail', 'path', {url_or_none}),
}),
'formats': formats,
})
post_info = {
**common_info,
'id': post_id,
'display_id': post_id,
**traverse_obj(post_data, {
'title': ('title', {str}),
'description': ('text', {clean_html}),
'like_count': ('likes', {int_or_none}),
'dislike_count': ('dislikes', {int_or_none}),
'comment_count': ('comments', {int_or_none}),
'thumbnail': ('thumbnail', 'path', {url_or_none}),
}),
}
if len(items) > 1:
return self.playlist_result(items, **post_info)
post_info.update(items[0])
return post_info
class FloatplaneChannelIE(InfoExtractor): class FloatplaneChannelIE(InfoExtractor):
_VALID_URL = r'https?://(?:(?:www|beta)\.)?floatplane\.com/channel/(?P<id>[\w-]+)/home(?:/(?P<channel>[\w-]+))?' _VALID_URL = r'https?://(?:(?:www|beta)\.)?floatplane\.com/channel/(?P<id>[\w-]+)/home(?:/(?P<channel>[\w-]+))?'
plugins/youtube_download/yt_dlp/extractor/francaisfacile.py
+1 -12
View File
@@ -1,9 +1,7 @@
import urllib.parse import urllib.parse
from .common import InfoExtractor from .common import InfoExtractor
from ..networking.exceptions import HTTPError
from ..utils import ( from ..utils import (
ExtractorError,
float_or_none, float_or_none,
url_or_none, url_or_none,
) )
@@ -58,16 +56,7 @@ class FrancaisFacileIE(InfoExtractor):
def _real_extract(self, url): def _real_extract(self, url):
display_id = urllib.parse.unquote(self._match_id(url)) display_id = urllib.parse.unquote(self._match_id(url))
webpage = self._download_webpage(url, display_id)
try: # yt-dlp's default user-agents are too old and blocked by the site
webpage = self._download_webpage(url, display_id, headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:136.0) Gecko/20100101 Firefox/136.0',
})
except ExtractorError as e:
if not isinstance(e.cause, HTTPError) or e.cause.status != 403:
raise
# Retry with impersonation if hardcoded UA is insufficient
webpage = self._download_webpage(url, display_id, impersonate=True)
data = self._search_json( data = self._search_json(
r'<script[^>]+\bdata-media-id=[^>]+\btype="application/json"[^>]*>', r'<script[^>]+\bdata-media-id=[^>]+\btype="application/json"[^>]*>',
plugins/youtube_download/yt_dlp/extractor/francetv.py
+25 -28
View File
@@ -1,4 +1,3 @@
import json
import re import re
import urllib.parse import urllib.parse
@@ -19,7 +18,11 @@ from ..utils import (
unsmuggle_url, unsmuggle_url,
url_or_none, url_or_none,
) )
from ..utils.traversal import find_element, traverse_obj from ..utils.traversal import (
find_element,
get_first,
traverse_obj,
)
class FranceTVBaseInfoExtractor(InfoExtractor): class FranceTVBaseInfoExtractor(InfoExtractor):
@@ -121,9 +124,10 @@ class FranceTVIE(InfoExtractor):
elif code := traverse_obj(dinfo, ('code', {int})): elif code := traverse_obj(dinfo, ('code', {int})):
if code == 2009: if code == 2009:
self.raise_geo_restricted(countries=self._GEO_COUNTRIES) self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
elif code in (2015, 2017): elif code in (2015, 2017, 2019):
# 2015: L'accès à cette vidéo est impossible. (DRM-only) # 2015: L'accès à cette vidéo est impossible. (DRM-only)
# 2017: Cette vidéo n'est pas disponible depuis le site web mobile (b/c DRM) # 2017: Cette vidéo n'est pas disponible depuis le site web mobile (b/c DRM)
# 2019: L'accès à cette vidéo est incompatible avec votre configuration. (DRM-only)
drm_formats = True drm_formats = True
continue continue
self.report_warning( self.report_warning(
@@ -258,7 +262,7 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
_TESTS = [{ _TESTS = [{
'url': 'https://www.france.tv/france-2/13h15-le-dimanche/140921-les-mysteres-de-jesus.html', 'url': 'https://www.france.tv/france-2/13h15-le-dimanche/140921-les-mysteres-de-jesus.html',
'info_dict': { 'info_dict': {
'id': 'ec217ecc-0733-48cf-ac06-af1347b849d1', # old: c5bda21d-2c6f-4470-8849-3d8327adb2ba' 'id': 'b2cf9fd8-e971-4757-8651-848f2772df61', # old: ec217ecc-0733-48cf-ac06-af1347b849d1
'ext': 'mp4', 'ext': 'mp4',
'title': '13h15, le dimanche... - Les mystères de Jésus', 'title': '13h15, le dimanche... - Les mystères de Jésus',
'timestamp': 1502623500, 'timestamp': 1502623500,
@@ -269,7 +273,7 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
'params': { 'params': {
'skip_download': True, 'skip_download': True,
}, },
'add_ie': [FranceTVIE.ie_key()], 'skip': 'Unfortunately, this video is no longer available',
}, { }, {
# geo-restricted # geo-restricted
'url': 'https://www.france.tv/enfants/six-huit-ans/foot2rue/saison-1/3066387-duel-au-vieux-port.html', 'url': 'https://www.france.tv/enfants/six-huit-ans/foot2rue/saison-1/3066387-duel-au-vieux-port.html',
@@ -287,7 +291,7 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
'thumbnail': r're:^https?://.*\.jpg$', 'thumbnail': r're:^https?://.*\.jpg$',
'duration': 1441, 'duration': 1441,
}, },
'skip': 'No longer available', 'skip': 'Unfortunately, this video is no longer available',
}, { }, {
# geo-restricted livestream (workflow == 'token-akamai') # geo-restricted livestream (workflow == 'token-akamai')
'url': 'https://www.france.tv/france-4/direct.html', 'url': 'https://www.france.tv/france-4/direct.html',
@@ -308,6 +312,19 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
'live_status': 'is_live', 'live_status': 'is_live',
}, },
'params': {'skip_download': 'livestream'}, 'params': {'skip_download': 'livestream'},
}, {
# Not geo-restricted
'url': 'https://www.france.tv/france-2/la-maison-des-maternelles/5574051-nous-sommes-amis-et-nous-avons-fait-un-enfant-ensemble.html',
'info_dict': {
'id': 'b448bfe4-9fe7-11ee-97d8-2ba3426fa3df',
'ext': 'mp4',
'title': 'Nous sommes amis et nous avons fait un enfant ensemble - Émission du jeudi 21 décembre 2023',
'duration': 1065,
'thumbnail': r're:https?://.+/.+\.jpg',
'timestamp': 1703147921,
'upload_date': '20231221',
},
'params': {'skip_download': 'm3u8'},
}, { }, {
# france3 # france3
'url': 'https://www.france.tv/france-3/des-chiffres-et-des-lettres/139063-emission-du-mardi-9-mai-2017.html', 'url': 'https://www.france.tv/france-3/des-chiffres-et-des-lettres/139063-emission-du-mardi-9-mai-2017.html',
@@ -342,31 +359,11 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
'only_matching': True, 'only_matching': True,
}] }]
# XXX: For parsing next.js v15+ data; see also yt_dlp.extractor.goplay
def _find_json(self, s):
return self._search_json(
r'\w+\s*:\s*', s, 'next js data', None, contains_pattern=r'\[(?s:.+)\]', default=None)
def _real_extract(self, url): def _real_extract(self, url):
display_id = self._match_id(url) display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id) webpage = self._download_webpage(url, display_id)
nextjs_data = self._search_nextjs_v13_data(webpage, display_id)
nextjs_data = traverse_obj( video_id = get_first(nextjs_data, ('options', 'id', {str}))
re.findall(r'<script[^>]*>\s*self\.__next_f\.push\(\s*(\[.+?\])\s*\);?\s*</script>', webpage),
(..., {json.loads}, ..., {self._find_json}, ..., 'children', ..., ..., 'children', ..., ..., 'children'))
if traverse_obj(nextjs_data, (..., ..., 'children', ..., 'isLive', {bool}, any)):
# For livestreams we need the id of the stream instead of the currently airing episode id
video_id = traverse_obj(nextjs_data, (
..., ..., 'children', ..., 'children', ..., 'children', ..., 'children', ..., ...,
'children', ..., ..., 'children', ..., ..., 'children', (..., (..., ...)),
'options', 'id', {str}, any))
else:
video_id = traverse_obj(nextjs_data, (
..., ..., ..., 'children',
lambda _, v: v['video']['url'] == urllib.parse.urlparse(url).path,
'video', ('playerReplayId', 'siId'), {str}, any))
if not video_id: if not video_id:
raise ExtractorError('Unable to extract video ID') raise ExtractorError('Unable to extract video ID')
plugins/youtube_download/yt_dlp/extractor/fujitv.py
+1 -1
View File
@@ -56,7 +56,7 @@ class FujiTVFODPlus7IE(InfoExtractor):
fmt, subs = self._extract_m3u8_formats_and_subtitles(src['url'], video_id, 'ts') fmt, subs = self._extract_m3u8_formats_and_subtitles(src['url'], video_id, 'ts')
for f in fmt: for f in fmt:
f.update(dict(zip(('height', 'width'), f.update(dict(zip(('height', 'width'),
self._BITRATE_MAP.get(f.get('tbr'), ())))) self._BITRATE_MAP.get(f.get('tbr'), ()), strict=False)))
formats.extend(fmt) formats.extend(fmt)
subtitles = self._merge_subtitles(subtitles, subs) subtitles = self._merge_subtitles(subtitles, subs)
plugins/youtube_download/yt_dlp/extractor/gamespot.py
+2 -2
View File
@@ -1,9 +1,9 @@
import urllib.parse import urllib.parse
from .once import OnceIE from .common import InfoExtractor
class GameSpotIE(OnceIE): class GameSpotIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?gamespot\.com/(?:video|article|review)s/(?:[^/]+/\d+-|embed/)(?P<id>\d+)' _VALID_URL = r'https?://(?:www\.)?gamespot\.com/(?:video|article|review)s/(?:[^/]+/\d+-|embed/)(?P<id>\d+)'
_TESTS = [{ _TESTS = [{
'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/', 'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
plugins/youtube_download/yt_dlp/extractor/generic.py
+501 -2100
View File
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More