itdominator/SolarFM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Upgraded youtube_download plugin

Browse Source
This commit is contained in:
itdominator
2026-05-26 20:50:58 -05:00
parent d55bc3ae97
commit 38ea00ec8f
87 changed files with 3385 additions and 1424 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/youtube_download/yt_dlp/extractor/_extractors.py
+52 -36
View File
@@ -1,32 +1,4 @@
# flake8: noqa: F401
# isort: off
from .youtube import ( # Youtube is moved to the top to improve performance
YoutubeIE,
YoutubeClipIE,
YoutubeFavouritesIE,
YoutubeNotificationsIE,
YoutubeHistoryIE,
YoutubeTabIE,
YoutubeLivestreamEmbedIE,
YoutubePlaylistIE,
YoutubeRecommendedIE,
YoutubeSearchDateIE,
YoutubeSearchIE,
YoutubeSearchURLIE,
YoutubeMusicSearchURLIE,
YoutubeSubscriptionsIE,
YoutubeTruncatedIDIE,
YoutubeTruncatedURLIE,
YoutubeYtBeIE,
YoutubeYtUserIE,
YoutubeWatchLaterIE,
YoutubeShortsAudioPivotIE,
YoutubeConsentRedirectIE,
)
# isort: on
from .abc import (
ABCIE,
ABCIViewIE,
@@ -339,8 +311,10 @@ from .canalsurmas import CanalsurmasIE
from .caracoltv import CaracolTvPlayIE
from .cbc import (
CBCIE,
CBCGemContentIE,
CBCGemIE,
CBCGemLiveIE,
CBCGemOlympicsIE,
CBCGemPlaylistIE,
CBCListenIE,
CBCPlayerIE,
@@ -592,7 +566,10 @@ from .eroprofile import (
EroProfileAlbumIE,
EroProfileIE,
)
from .err import ERRJupiterIE
from .err import (
ERRArhiivIE,
ERRJupiterIE,
)
from .ertgr import (
ERTFlixCodenameIE,
ERTFlixIE,
@@ -695,10 +672,6 @@ from .frontendmasters import (
FrontendMastersIE,
FrontendMastersLessonIE,
)
from .frontro import (
TheChosenGroupIE,
TheChosenIE,
)
from .fujitv import FujiTVFODPlus7IE
from .funk import FunkIE
from .funker530 import Funker530IE
@@ -1054,6 +1027,10 @@ from .livestream import (
)
from .livestreamfails import LivestreamfailsIE
from .lnk import LnkIE
from .locipo import (
LocipoIE,
LocipoPlaylistIE,
)
from .loco import LocoIE
from .loom import (
LoomFolderIE,
@@ -1096,6 +1073,7 @@ from .markiza import (
)
from .massengeschmacktv import MassengeschmackTVIE
from .masters import MastersIE
from .matchitv import MatchiTVIE
from .matchtv import MatchTVIE
from .mave import (
MaveChannelIE,
@@ -1810,7 +1788,10 @@ from .safari import (
from .saitosan import SaitosanIE
from .samplefocus import SampleFocusIE
from .sapo import SapoIE
from .sauceplus import SaucePlusIE
from .sauceplus import (
SaucePlusChannelIE,
SaucePlusIE,
)
from .sbs import SBSIE
from .sbscokr import (
SBSCoKrAllvodProgramIE,
@@ -2078,6 +2059,10 @@ from .tenplay import (
from .testurl import TestURLIE
from .tf1 import TF1IE
from .tfo import TFOIE
from .thechosen import (
TheChosenGroupIE,
TheChosenIE,
)
from .theguardian import (
TheGuardianPodcastIE,
TheGuardianPodcastPlaylistIE,
@@ -2199,11 +2184,15 @@ from .tvc import (
TVCIE,
TVCArticleIE,
)
from .tver import TVerIE
from .tver import (
TVerIE,
TVerOlympicIE,
)
from .tvigle import TvigleIE
from .tviplayer import TVIPlayerIE
from .tvn24 import TVN24IE
from .tvnoe import TVNoeIE
from .tvo import TvoIE
from .tvopengr import (
TVOpenGrEmbedIE,
TVOpenGrWatchIE,
@@ -2368,6 +2357,7 @@ from .vimm import (
)
from .viously import ViouslyIE
from .viqeo import ViqeoIE
from .visir import VisirIE
from .viu import (
ViuIE,
ViuOTTIE,
@@ -2388,7 +2378,11 @@ from .voicy import (
VoicyChannelIE,
VoicyIE,
)
from .volejtv import VolejTVIE
from .volejtv import (
VolejTVCategoryPlaylistIE,
VolejTVClubPlaylistIE,
VolejTVIE,
)
from .voxmedia import (
VoxMediaIE,
VoxMediaVolumeIE,
@@ -2551,6 +2545,28 @@ from .youporn import (
YouPornTagIE,
YouPornVideosIE,
)
from .youtube import (
YoutubeClipIE,
YoutubeConsentRedirectIE,
YoutubeFavouritesIE,
YoutubeHistoryIE,
YoutubeIE,
YoutubeLivestreamEmbedIE,
YoutubeMusicSearchURLIE,
YoutubeNotificationsIE,
YoutubePlaylistIE,
YoutubeRecommendedIE,
YoutubeSearchIE,
YoutubeSearchURLIE,
YoutubeShortsAudioPivotIE,
YoutubeSubscriptionsIE,
YoutubeTabIE,
YoutubeTruncatedIDIE,
YoutubeTruncatedURLIE,
YoutubeWatchLaterIE,
YoutubeYtBeIE,
YoutubeYtUserIE,
)
from .zaiko import (
ZaikoETicketIE,
ZaikoIE,
plugins/youtube_download/yt_dlp/extractor/aenetworks.py
+61 -18
View File
@@ -5,10 +5,12 @@ from ..utils import (
ExtractorError,
GeoRestrictedError,
int_or_none,
make_archive_id,
remove_start,
traverse_obj,
update_url_query,
url_or_none,
)
from ..utils.traversal import traverse_obj
class AENetworksBaseIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
@@ -29,6 +31,19 @@ class AENetworksBaseIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
'historyvault.com': (None, 'historyvault', None),
'biography.com': (None, 'biography', None),
}
_GRAPHQL_QUERY = '''
query getUserVideo($videoId: ID!) {
video(id: $videoId) {
title
publicUrl
programId
tvSeasonNumber
tvSeasonEpisodeNumber
series {
title
}
}
}'''
def _extract_aen_smil(self, smil_url, video_id, auth=None):
query = {
@@ -73,19 +88,39 @@ class AENetworksBaseIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
def _extract_aetn_info(self, domain, filter_key, filter_value, url):
requestor_id, brand, software_statement = self._DOMAIN_MAP[domain]
if filter_key == 'canonical':
webpage = self._download_webpage(url, filter_value)
graphql_video_id = self._search_regex(
r'<meta\b[^>]+\bcontent="[^"]*\btpid/(\d+)"', webpage, 'id',
default=None) or self._html_search_meta('videoId', webpage, 'GraphQL video ID', fatal=True)
else:
graphql_video_id = filter_value
result = self._download_json(
f'https://feeds.video.aetnd.com/api/v2/{brand}/videos',
filter_value, query={f'filter[{filter_key}]': filter_value})
result = traverse_obj(
result, ('results',
lambda k, v: k == 0 and v[filter_key] == filter_value),
get_all=False)
if not result:
'https://yoga.appsvcs.aetnd.com/', graphql_video_id,
query={
'brand': brand,
'mode': 'live',
'platform': 'web',
},
data=json.dumps({
'operationName': 'getUserVideo',
'variables': {
'videoId': graphql_video_id,
},
'query': self._GRAPHQL_QUERY,
}).encode(),
headers={
'Content-Type': 'application/json',
})
result = traverse_obj(result, ('data', 'video', {dict}))
media_url = traverse_obj(result, ('publicUrl', {url_or_none}))
if not media_url:
raise ExtractorError('Show not found in A&E feed (too new?)', expected=True,
video_id=remove_start(filter_value, '/'))
title = result['title']
video_id = result['id']
media_url = result['publicUrl']
video_id = result['programId']
theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
r'https?://link\.theplatform\.com/s/([^?]+)', media_url, 'theplatform_path'), video_id)
info = self._parse_theplatform_metadata(theplatform_metadata)
@@ -100,9 +135,13 @@ class AENetworksBaseIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
info.update(self._extract_aen_smil(media_url, video_id, auth))
info.update({
'title': title,
'series': result.get('seriesName'),
'season_number': int_or_none(result.get('tvSeasonNumber')),
'episode_number': int_or_none(result.get('tvSeasonEpisodeNumber')),
'display_id': graphql_video_id,
'_old_archive_ids': [make_archive_id(self, graphql_video_id)],
**traverse_obj(result, {
'series': ('series', 'title', {str}),
'season_number': ('tvSeasonNumber', {int_or_none}),
'episode_number': ('tvSeasonEpisodeNumber', {int_or_none}),
}),
})
return info
@@ -116,7 +155,7 @@ class AENetworksIE(AENetworksBaseIE):
(?:shows/[^/?#]+/)?videos/[^/?#]+
)'''
_TESTS = [{
'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
'url': 'https://www.history.com/shows/mountain-men/season-1/episode-1',
'info_dict': {
'id': '22253814',
'ext': 'mp4',
@@ -139,11 +178,11 @@ class AENetworksIE(AENetworksBaseIE):
},
'params': {'skip_download': 'm3u8'},
'add_ie': ['ThePlatform'],
'skip': 'Geo-restricted - This content is not available in your location.',
'skip': 'This content requires a valid, unexpired auth token',
}, {
'url': 'http://www.aetv.com/shows/duck-dynasty/season-9/episode-1',
'url': 'https://www.aetv.com/shows/duck-dynasty/season-9/episode-1',
'info_dict': {
'id': '600587331957',
'id': '147486',
'ext': 'mp4',
'title': 'Inlawful Entry',
'description': 'md5:57c12115a2b384d883fe64ca50529e08',
@@ -160,6 +199,8 @@ class AENetworksIE(AENetworksBaseIE):
'season_number': 9,
'series': 'Duck Dynasty',
'age_limit': 0,
'display_id': '600587331957',
'_old_archive_ids': ['aenetworks 600587331957'],
},
'params': {'skip_download': 'm3u8'},
'add_ie': ['ThePlatform'],
@@ -186,6 +227,7 @@ class AENetworksIE(AENetworksBaseIE):
},
'params': {'skip_download': 'm3u8'},
'add_ie': ['ThePlatform'],
'skip': '404 Not Found',
}, {
'url': 'https://www.aetv.com/specials/hunting-jonbenets-killer-the-untold-story',
'info_dict': {
@@ -209,6 +251,7 @@ class AENetworksIE(AENetworksBaseIE):
},
'params': {'skip_download': 'm3u8'},
'add_ie': ['ThePlatform'],
'skip': 'This content requires a valid, unexpired auth token',
}, {
'url': 'http://www.fyi.tv/shows/tiny-house-nation/season-1/episode-8',
'only_matching': True,
@@ -259,7 +302,7 @@ class AENetworksListBaseIE(AENetworksBaseIE):
domain, slug = self._match_valid_url(url).groups()
_, brand, _ = self._DOMAIN_MAP[domain]
playlist = self._call_api(self._RESOURCE, slug, brand, self._FIELDS)
base_url = f'http://watch.{domain}'
base_url = f'https://watch.{domain}'
entries = []
for item in (playlist.get(self._ITEMS_KEY) or []):
plugins/youtube_download/yt_dlp/extractor/afreecatv.py
+118 -76
View File
@@ -1,5 +1,6 @@
import datetime as dt
import functools
import time
from .common import InfoExtractor
from ..networking import Request
@@ -16,7 +17,23 @@ from ..utils import (
urlencode_postdata,
urljoin,
)
from ..utils.traversal import traverse_obj
from ..utils.traversal import require, traverse_obj
def _cloudfront_auth_request(m3u8_url, strm_id, video_id, referer_url):
return Request(
'https://live.sooplive.com/api/private_auth.php',
method='POST',
headers={
'Referer': referer_url,
'Origin': 'https://vod.sooplive.com',
},
data=urlencode_postdata({
'type': 'vod',
'strm_id': strm_id,
'title_no': video_id,
'url': m3u8_url,
}))
class AfreecaTVBaseIE(InfoExtractor):
@@ -34,17 +51,17 @@ class AfreecaTVBaseIE(InfoExtractor):
}
response = self._download_json(
'https://login.sooplive.co.kr/app/LoginAction.php', None,
'https://login.sooplive.com/app/LoginAction.php', None,
'Logging in', data=urlencode_postdata(login_form))
_ERRORS = {
-4: 'Your account has been suspended due to a violation of our terms and policies.',
-5: 'https://member.sooplive.co.kr/app/user_delete_progress.php',
-6: 'https://login.sooplive.co.kr/membership/changeMember.php',
-5: 'https://member.sooplive.com/app/user_delete_progress.php',
-6: 'https://login.sooplive.com/membership/changeMember.php',
-8: "Hello! Soop here.\nThe username you have entered belongs to \n an account that requires a legal guardian's consent. \nIf you wish to use our services without restriction, \nplease make sure to go through the necessary verification process.",
-9: 'https://member.sooplive.co.kr/app/pop_login_block.php',
-11: 'https://login.sooplive.co.kr/afreeca/second_login.php',
-12: 'https://member.sooplive.co.kr/app/user_security.php',
-9: 'https://member.sooplive.com/app/pop_login_block.php',
-11: 'https://login.sooplive.com/afreeca/second_login.php',
-12: 'https://member.sooplive.com/app/user_security.php',
0: 'The username does not exist or you have entered the wrong password.',
-1: 'The username does not exist or you have entered the wrong password.',
-3: 'You have entered your username/password incorrectly.',
@@ -62,7 +79,7 @@ class AfreecaTVBaseIE(InfoExtractor):
def _call_api(self, endpoint, display_id, data=None, headers=None, query=None):
return self._download_json(Request(
f'https://api.m.sooplive.co.kr/{endpoint}',
f'https://api.m.sooplive.com/{endpoint}',
data=data, headers=headers, query=query,
extensions={'legacy_ssl': True}), display_id,
'Downloading API JSON', 'Unable to download API JSON')
@@ -78,36 +95,36 @@ class AfreecaTVBaseIE(InfoExtractor):
class AfreecaTVIE(AfreecaTVBaseIE):
IE_NAME = 'soop'
IE_DESC = 'sooplive.co.kr'
_VALID_URL = r'https?://vod\.(?:sooplive\.co\.kr|afreecatv\.com)/(?:PLAYER/STATION|player)/(?P<id>\d+)/?(?:$|[?#&])'
IE_DESC = 'sooplive.com'
_VALID_URL = r'https?://vod\.sooplive\.com/(?:PLAYER/STATION|player)/(?P<id>\d+)/?(?:$|[?#&])'
_TESTS = [{
'url': 'https://vod.sooplive.co.kr/player/96753363',
'url': 'https://vod.sooplive.com/player/192805325',
'info_dict': {
'id': '20230108_9FF5BEE1_244432674_1',
'id': '20260414_1B44E53B_293230967_1',
'ext': 'mp4',
'uploader_id': 'rlantnghks',
'uploader': '페이즈으',
'duration': 10840,
'thumbnail': r're:https?://videoimg\.(?:sooplive\.co\.kr|afreecatv\.com)/.+',
'upload_date': '20230108',
'timestamp': 1673186405,
'title': '젠지 페이즈',
'duration': 10869,
'thumbnail': r're:https?://videoimg\.sooplive\.com/.+',
'upload_date': '20260414',
'timestamp': 1776174982,
'title': 'T1 Peyz [CC]',
},
'params': {
'skip_download': True,
},
}, {
# non standard key
'url': 'http://vod.sooplive.co.kr/PLAYER/STATION/20515605',
'url': 'https://vod.sooplive.com/PLAYER/STATION/20515605',
'info_dict': {
'id': '20170411_BE689A0E_190960999_1_2_h',
'id': 'BE689A0E_190960999_1_2_A',
'ext': 'mp4',
'title': '혼자사는여자집',
'thumbnail': r're:https?://(?:video|st)img\.(?:sooplive\.co\.kr|afreecatv\.com)/.+',
'thumbnail': r're:https?://(?:video|st)img\.(?:sooplive\.co\.kr|sooplive\.com|afreecatv\.com)/.+',
'uploader': '♥이슬이',
'uploader_id': 'dasl8121',
'upload_date': '20170411',
'timestamp': 1491929865,
'timestamp': 1491897465,
'duration': 213,
},
'params': {
@@ -115,32 +132,32 @@ class AfreecaTVIE(AfreecaTVBaseIE):
},
}, {
# adult content
'url': 'https://vod.sooplive.co.kr/player/97267690',
'url': 'https://vod.sooplive.com/player/191612613',
'info_dict': {
'id': '20180327_27901457_202289533_1',
'id': '20260403_A4534670_292914441_1',
'ext': 'mp4',
'title': '[생]빨개요♥ (part 1)',
'thumbnail': r're:https?://(?:video|st)img\.(?:sooplive\.co\.kr|afreecatv\.com)/.+',
'uploader': '[SA]서아',
'uploader_id': 'bjdyrksu',
'upload_date': '20180327',
'duration': 3601,
'title': '하이',
'thumbnail': r're:https?://(?:video|st)img\.sooplive\.com/.+',
'uploader': '세림잉',
'uploader_id': 'serimm',
'upload_date': '20260402',
'duration': 7015,
},
'params': {
'skip_download': True,
},
'skip': 'The VOD does not exist',
'skip': 'Needs login to view',
}, {
# adult content
'url': 'https://vod.sooplive.co.kr/player/70395877',
'url': 'https://vod.sooplive.com/player/70395877',
'only_matching': True,
}, {
# subscribers only
'url': 'https://vod.sooplive.co.kr/player/104647403',
'url': 'https://vod.sooplive.com/player/104647403',
'only_matching': True,
}, {
# private
'url': 'https://vod.sooplive.co.kr/player/81669846',
'url': 'https://vod.sooplive.com/player/81669846',
'only_matching': True,
}]
@@ -153,6 +170,13 @@ class AfreecaTVIE(AfreecaTVBaseIE):
'nApiLevel': 10,
}))['data']
initial_refresh_time = 0
strm_id = None
# For subscriber-only VODs, we need to call private_auth.php to get CloudFront cookies
needs_private_auth = traverse_obj(data, ('sub_upload_type', {str}))
if needs_private_auth:
strm_id = traverse_obj(data, ('bj_id', {str}, {require('stream ID')}))
error_code = traverse_obj(data, ('code', {int}))
if error_code == -6221:
raise ExtractorError('The VOD does not exist', expected=True)
@@ -172,9 +196,23 @@ class AfreecaTVIE(AfreecaTVBaseIE):
traverse_obj(data, ('files', lambda _, v: url_or_none(v['file']))), start=1):
file_url = file_element['file']
if determine_ext(file_url) == 'm3u8':
if needs_private_auth:
self._request_webpage(
_cloudfront_auth_request(file_url, strm_id, video_id, url),
video_id, 'Requesting CloudFront cookies', 'Failed to get CloudFront cookies')
initial_refresh_time = time.time()
formats = self._extract_m3u8_formats(
file_url, video_id, 'mp4', m3u8_id='hls',
note=f'Downloading part {file_num} m3u8 information')
if needs_private_auth:
for fmt in formats:
fmt['protocol'] = 'soopvod'
fmt['_cookie_refresh_params'] = {
'm3u8_url': file_url,
'strm_id': strm_id,
'video_id': video_id,
'_last_refresh': initial_refresh_time,
}
else:
formats = [{
'url': file_url,
@@ -217,10 +255,10 @@ class AfreecaTVIE(AfreecaTVBaseIE):
class AfreecaTVCatchStoryIE(AfreecaTVBaseIE):
IE_NAME = 'soop:catchstory'
IE_DESC = 'sooplive.co.kr catch story'
_VALID_URL = r'https?://vod\.(?:sooplive\.co\.kr|afreecatv\.com)/player/(?P<id>\d+)/catchstory'
IE_DESC = 'sooplive.com catch story'
_VALID_URL = r'https?://vod\.sooplive\.com/player/(?P<id>\d+)/catchstory'
_TESTS = [{
'url': 'https://vod.sooplive.co.kr/player/103247/catchstory',
'url': 'https://vod.sooplive.com/player/103247/catchstory',
'info_dict': {
'id': '103247',
},
@@ -253,10 +291,10 @@ class AfreecaTVCatchStoryIE(AfreecaTVBaseIE):
class AfreecaTVLiveIE(AfreecaTVBaseIE):
IE_NAME = 'soop:live'
IE_DESC = 'sooplive.co.kr livestreams'
_VALID_URL = r'https?://play\.(?:sooplive\.co\.kr|afreecatv\.com)/(?P<id>[^/?#]+)(?:/(?P<bno>\d+))?'
IE_DESC = 'sooplive.com livestreams'
_VALID_URL = r'https?://play\.sooplive\.com/(?P<id>[^/?#]+)(?:/(?P<bno>\d+))?'
_TESTS = [{
'url': 'https://play.sooplive.co.kr/pyh3646/237852185',
'url': 'https://play.sooplive.com/pyh3646/237852185',
'info_dict': {
'id': '237852185',
'ext': 'mp4',
@@ -268,30 +306,33 @@ class AfreecaTVLiveIE(AfreecaTVBaseIE):
},
'skip': 'Livestream has ended',
}, {
'url': 'https://play.sooplive.co.kr/pyh3646/237852185',
'url': 'https://play.sooplive.com/pyh3646/237852185',
'only_matching': True,
}, {
'url': 'https://play.sooplive.co.kr/pyh3646',
'url': 'https://play.sooplive.com/pyh3646',
'only_matching': True,
}]
_LIVE_API_URL = 'https://live.sooplive.co.kr/afreeca/player_live_api.php'
_LIVE_API_URL = 'https://live.sooplive.com/afreeca/player_live_api.php'
_WORKING_CDNS = [
'gcp_cdn', # live-global-cdn-v02.sooplive.co.kr
'gs_cdn_pc_app', # pc-app.stream.sooplive.co.kr
'gs_cdn_mobile_web', # mobile-web.stream.sooplive.co.kr
'gs_cdn_pc_web', # pc-web.stream.sooplive.co.kr
'gcp_cdn', # live-global-cdn-v02.sooplive.com
'gs_cdn_mobile_web', # mobile-web.stream.sooplive.com
'gs_cdn_pc_web', # pc-web.stream.sooplive.com
'lg_cdn_pc_web', # live-pcweb-kr-cdn-z02.sooplive.com
'lg_cdn_mobile_web', # live-mobileweb-kr-cdn-z02.sooplive.com
'azure_cdn', # live-global-cdn-v02.sooplive.com
'aws_cf', # live-global-cdn-v02.sooplive.com
]
_BAD_CDNS = [
'gs_cdn', # chromecast.afreeca.gscdn.com (cannot resolve)
'gs_cdn_chromecast', # chromecast.stream.sooplive.co.kr (HTTP Error 400)
'azure_cdn', # live-global-cdn-v01.sooplive.co.kr (cannot resolve)
'aws_cf', # live-global-cdn-v03.sooplive.co.kr (cannot resolve)
'kt_cdn', # kt.stream.sooplive.co.kr (HTTP Error 400)
'gs_cdn', # chromecast.stream.sooplive.com (cannot resolve)
'gs_cdn_chromecast', # chromecast.stream.sooplive.com (HTTP Error 400)
'lg_cdn_chromecast', # live-chromecast-kr-cdn-z02.sooplive.com (HTTP Error 403)
'gs_cdn_pc_app', # pc-app.stream.sooplive.co.kr (HTTP Error 400) Might not exist anymore
'kt_cdn', # kt.stream.sooplive.co.kr (HTTP Error 400) Might not exist anymore
]
def _extract_formats(self, channel_info, broadcast_no, aid):
stream_base_url = channel_info.get('RMD') or 'https://livestream-manager.sooplive.co.kr'
stream_base_url = channel_info.get('RMD') or 'https://livestream-manager.sooplive.com'
# If user has not passed CDN IDs, try API-provided CDN ID followed by other working CDN IDs
default_cdn_ids = orderedSet([
@@ -311,7 +352,7 @@ class AfreecaTVLiveIE(AfreecaTVBaseIE):
try:
return self._extract_m3u8_formats(
m3u8_url, broadcast_no, 'mp4', m3u8_id='hls', query={'aid': aid},
headers={'Referer': 'https://play.sooplive.co.kr/'})
headers={'Referer': 'https://play.sooplive.com/'})
except ExtractorError as e:
if attempt == len(cdn_ids):
raise
@@ -362,7 +403,7 @@ class AfreecaTVLiveIE(AfreecaTVBaseIE):
formats = self._extract_formats(channel_info, broadcast_no, aid)
station_info = traverse_obj(self._download_json(
'https://st.sooplive.co.kr/api/get_station_status.php', broadcast_no,
'https://st.sooplive.com/api/get_station_status.php', broadcast_no,
'Downloading channel metadata', 'Unable to download channel metadata',
query={'szBjId': broadcaster_id}, fatal=False), {dict}) or {}
@@ -380,50 +421,51 @@ class AfreecaTVLiveIE(AfreecaTVBaseIE):
class AfreecaTVUserIE(AfreecaTVBaseIE):
IE_NAME = 'soop:user'
_VALID_URL = r'https?://ch\.(?:sooplive\.co\.kr|afreecatv\.com)/(?P<id>[^/?#]+)/vods/?(?P<slug_type>[^/?#]+)?'
_VALID_URL = r'https?://(?:www\.)?sooplive\.com/station/(?P<id>[^/?#]+)/vod/?(?P<slug_type>[^/?#]+)?'
_TESTS = [{
'url': 'https://ch.sooplive.co.kr/ryuryu24/vods/review',
'url': 'https://www.sooplive.com/station/devil0108/vod/review',
'info_dict': {
'_type': 'playlist',
'id': 'ryuryu24',
'title': 'ryuryu24 - review',
'id': 'devil0108',
'title': 'devil0108 - review',
},
'playlist_count': 218,
'playlist_mincount': 300,
}, {
'url': 'https://ch.sooplive.co.kr/parang1995/vods/highlight',
'url': 'https://www.sooplive.com/station/devil0108/vod',
'info_dict': {
'_type': 'playlist',
'id': 'devil0108',
'title': 'devil0108 - all',
},
'playlist_mincount': 300,
}, {
'url': 'https://www.sooplive.com/station/parang1995/vod/clip',
'info_dict': {
'_type': 'playlist',
'id': 'parang1995',
'title': 'parang1995 - highlight',
'title': 'parang1995 - clip',
},
'playlist_count': 997,
'playlist_mincount': 300,
}, {
'url': 'https://ch.sooplive.co.kr/ryuryu24/vods',
'url': 'https://www.sooplive.com/station/phonics1/vod/normal',
'info_dict': {
'_type': 'playlist',
'id': 'ryuryu24',
'title': 'ryuryu24 - all',
'id': 'phonics1',
'title': 'phonics1 - normal',
},
'playlist_count': 221,
}, {
'url': 'https://ch.sooplive.co.kr/ryuryu24/vods/balloonclip',
'info_dict': {
'_type': 'playlist',
'id': 'ryuryu24',
'title': 'ryuryu24 - balloonclip',
},
'playlist_count': 0,
'playlist_mincount': 300,
}]
_PER_PAGE = 60
def _fetch_page(self, user_id, user_type, page):
page += 1
info = self._download_json(f'https://chapi.sooplive.co.kr/api/{user_id}/vods/{user_type}', user_id,
info = self._download_json(f'https://chapi.sooplive.com/api/{user_id}/vods/{user_type}', user_id,
query={'page': page, 'per_page': self._PER_PAGE, 'orderby': 'reg_date'},
note=f'Downloading {user_type} video page {page}')
for item in info['data']:
yield self.url_result(
f'https://vod.sooplive.co.kr/player/{item["title_no"]}/', AfreecaTVIE, item['title_no'])
f'https://vod.sooplive.com/player/{item["title_no"]}/',
AfreecaTVIE, item['title_no'], item.get('title_name'))
def _real_extract(self, url):
user_id, user_type = self._match_valid_url(url).group('id', 'slug_type')
plugins/youtube_download/yt_dlp/extractor/applepodcasts.py
+15 -15
View File
@@ -11,18 +11,18 @@ from ..utils.traversal import traverse_obj
class ApplePodcastsIE(InfoExtractor):
_VALID_URL = r'https?://podcasts\.apple\.com/(?:[^/]+/)?podcast(?:/[^/]+){1,2}.*?\bi=(?P<id>\d+)'
_TESTS = [{
'url': 'https://podcasts.apple.com/us/podcast/ferreck-dawn-to-the-break-of-dawn-117/id1625658232?i=1000665010654',
'md5': '82cc219b8cc1dcf8bfc5a5e99b23b172',
'url': 'https://podcasts.apple.com/us/podcast/urbana-podcast-724-by-david-penn/id1531349107?i=1000748574256',
'md5': 'f8a6f92735d0cfbd5e6a7294151e28d8',
'info_dict': {
'id': '1000665010654',
'ext': 'mp3',
'title': 'Ferreck Dawn - To The Break of Dawn 117',
'episode': 'Ferreck Dawn - To The Break of Dawn 117',
'description': 'md5:8c4f5c2c30af17ed6a98b0b9daf15b76',
'upload_date': '20240812',
'timestamp': 1723449600,
'duration': 3596,
'series': 'Ferreck Dawn - To The Break of Dawn',
'id': '1000748574256',
'ext': 'm4a',
'title': 'URBANA PODCAST 724 BY DAVID PENN',
'episode': 'URBANA PODCAST 724 BY DAVID PENN',
'description': 'md5:fec77bacba32db8c9b3dda5486ed085f',
'upload_date': '20260206',
'timestamp': 1770400801,
'duration': 3602,
'series': 'Urbana Radio Show',
'thumbnail': 're:.+[.](png|jpe?g|webp)',
},
}, {
@@ -57,22 +57,22 @@ class ApplePodcastsIE(InfoExtractor):
webpage = self._download_webpage(url, episode_id)
server_data = self._search_json(
r'<script [^>]*\bid=["\']serialized-server-data["\'][^>]*>', webpage,
'server data', episode_id, contains_pattern=r'\[{(?s:.+)}\]')[0]['data']
'server data', episode_id)['data'][0]['data']
model_data = traverse_obj(server_data, (
'headerButtonItems', lambda _, v: v['$kind'] == 'share' and v['modelType'] == 'EpisodeLockup',
'model', {dict}, any))
return {
'id': episode_id,
**self._json_ld(
traverse_obj(server_data, ('seoData', 'schemaContent', {dict}))
or self._yield_json_ld(webpage, episode_id, fatal=False), episode_id, fatal=False),
**traverse_obj(model_data, {
'title': ('title', {str}),
'description': ('summary', {clean_html}),
'url': ('playAction', 'episodeOffer', 'streamUrl', {clean_podcast_url}),
'timestamp': ('releaseDate', {parse_iso8601}),
'duration': ('duration', {int_or_none}),
'episode': ('title', {str}),
'episode_number': ('episodeNumber', {int_or_none}),
'series': ('showTitle', {str}),
}),
'thumbnail': self._og_search_thumbnail(webpage),
'vcodec': 'none',
plugins/youtube_download/yt_dlp/extractor/ard.py
+5 -5
View File
@@ -618,7 +618,7 @@ class ARDAudiothekBaseIE(InfoExtractor):
class ARDAudiothekIE(ARDAudiothekBaseIE):
_VALID_URL = r'https:?//(?:www\.)?ardaudiothek\.de/episode/(?P<id>urn:ard:(?:episode|section|extra):[a-f0-9]{16})'
_VALID_URL = r'https:?//(?:www\.)?ard(?:audiothek|sounds)\.de/episode/(?P<id>urn:ard:(?:episode|section|extra):[a-f0-9]{16})'
_TESTS = [{
'url': 'https://www.ardaudiothek.de/episode/urn:ard:episode:eabead1add170e93/',
@@ -649,13 +649,13 @@ class ARDAudiothekIE(ARDAudiothekBaseIE):
'description': 'md5:3552d571e1959754cff66c1da6c0fdae',
'display_id': 'urn:ard:section:855c7a53dac72e0a',
'timestamp': 1735629900,
'series': 'Auf der Spur – Die ARD Ermittlerkrimis',
'series': 'Auf der Spur - ARD Ermittlerkrimis',
'channel': 'ARD',
'episode': 'Episode 1',
'episode_number': 1,
},
}, {
'url': 'https://www.ardaudiothek.de/episode/urn:ard:extra:d2fe7303d2dcbf5d/',
'url': 'https://www.ardsounds.de/episode/urn:ard:extra:d2fe7303d2dcbf5d/',
'info_dict': {
'id': 'urn:ard:extra:d2fe7303d2dcbf5d',
'ext': 'mp3',
@@ -724,7 +724,7 @@ class ARDAudiothekIE(ARDAudiothekBaseIE):
class ARDAudiothekPlaylistIE(ARDAudiothekBaseIE):
_VALID_URL = r'https:?//(?:www\.)?ardaudiothek\.de/sendung/(?P<playlist>[\w-]+)/(?P<id>urn:ard:show:[a-f0-9]{16})'
_VALID_URL = r'https:?//(?:www\.)?ard(?:audiothek|sounds)\.de/sendung/(?P<playlist>[\w-]+)/(?P<id>urn:ard:show:[a-f0-9]{16})'
_TESTS = [{
'url': 'https://www.ardaudiothek.de/sendung/mia-insomnia/urn:ard:show:c405aa26d9a4060a/',
@@ -736,7 +736,7 @@ class ARDAudiothekPlaylistIE(ARDAudiothekBaseIE):
},
'playlist_mincount': 37,
}, {
'url': 'https://www.ardaudiothek.de/sendung/100-berlin/urn:ard:show:4d248e0806ce37bc/',
'url': 'https://www.ardsounds.de/sendung/100-berlin/urn:ard:show:4d248e0806ce37bc/',
'only_matching': True,
}]
plugins/youtube_download/yt_dlp/extractor/bandcamp.py
+7 -11
View File
@@ -439,31 +439,27 @@ class BandcampWeeklyIE(BandcampIE): # XXX: Do not subclass from concrete IE
def _real_extract(self, url):
show_id = self._match_id(url)
audio_data = self._download_json(
show_data = self._download_json(
'https://bandcamp.com/api/bcradio_api/1/get_show',
show_id, 'Downloading radio show JSON',
data=json.dumps({'id': show_id}).encode(),
headers={'Content-Type': 'application/json'})['radioShowAudio']
headers={'Content-Type': 'application/json'})
audio_data = show_data['compiledTrack']
stream_url = audio_data['streamUrl']
format_id = traverse_obj(stream_url, ({parse_qs}, 'enc', -1))
encoding, _, bitrate_str = (format_id or '').partition('-')
webpage = self._download_webpage(url, show_id, fatal=False)
metadata = traverse_obj(
self._extract_data_attr(webpage, show_id, 'blob', fatal=False),
('appData', 'shows', lambda _, v: str(v['showId']) == show_id, any)) or {}
series_title = audio_data.get('title') or metadata.get('title')
release_timestamp = unified_timestamp(audio_data.get('date')) or unified_timestamp(metadata.get('date'))
series_title = show_data.get('title')
release_timestamp = unified_timestamp(show_data.get('date'))
return {
'id': show_id,
'episode_id': show_id,
'title': join_nonempty(series_title, strftime_or_none(release_timestamp, '%Y-%m-%d'), delim=', '),
'series': series_title,
'thumbnail': format_field(metadata, 'imageId', 'https://f4.bcbits.com/img/%s_0.jpg', default=None),
'description': metadata.get('desc') or metadata.get('short_desc'),
'thumbnail': format_field(show_data, 'imageId', 'https://f4.bcbits.com/img/%s_0.jpg', default=None),
'description': show_data.get('description'),
'duration': float_or_none(audio_data.get('duration')),
'release_timestamp': release_timestamp,
'formats': [{
plugins/youtube_download/yt_dlp/extractor/bilibili.py
+1 -1
View File
@@ -124,7 +124,7 @@ class BilibiliBaseIE(InfoExtractor):
**traverse_obj(play_info, {
'quality': ('quality', {int_or_none}),
'format_id': ('quality', {str_or_none}),
'format_note': ('quality', {lambda x: format_names.get(x)}),
'format_note': ('quality', {format_names.get}),
'duration': ('timelength', {float_or_none(scale=1000)}),
}),
**parse_resolution(format_names.get(play_info.get('quality'))),
plugins/youtube_download/yt_dlp/extractor/boosty.py
+45 -16
View File
@@ -21,21 +21,44 @@ class BoostyIE(InfoExtractor):
'url': 'https://boosty.to/kuplinov/posts/e55d050c-e3bb-4873-a7db-ac7a49b40c38',
'info_dict': {
'id': 'd7473824-352e-48e2-ae53-d4aa39459968',
'title': 'phasma_3',
'title': 'Бан? А! Бан! (Phasmophobia)',
'alt_title': 'Бан? А! Бан! (Phasmophobia)',
'channel': 'Kuplinov',
'channel_id': '7958701',
'timestamp': 1655031975,
'upload_date': '20220612',
'release_timestamp': 1655049000,
'release_date': '20220612',
'modified_timestamp': 1668680993,
'modified_date': '20221117',
'modified_timestamp': 1743328648,
'modified_date': '20250330',
'tags': ['куплинов', 'phasmophobia'],
'like_count': int,
'ext': 'mp4',
'duration': 105,
'view_count': int,
'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
'thumbnail': r're:^https://iv\.okcdn\.ru/videoPreview\?',
},
}, {
# single ok_video with truncated title
'url': 'https://boosty.to/kuplinov/posts/cc09b7f9-121e-40b8-9392-4a075ef2ce53',
'info_dict': {
'id': 'fb5ea762-6303-4557-9a17-157947326810',
'title': 'Какая там активность была? Не слышу! Повтори еще пару раз! (Phas',
'alt_title': 'Какая там активность была? Не слышу! Повтори еще пару раз! (Phasmophobia)',
'channel': 'Kuplinov',
'channel_id': '7958701',
'timestamp': 1655031930,
'upload_date': '20220612',
'release_timestamp': 1655048400,
'release_date': '20220612',
'modified_timestamp': 1743328616,
'modified_date': '20250330',
'tags': ['куплинов', 'phasmophobia'],
'like_count': int,
'ext': 'mp4',
'duration': 39,
'view_count': int,
'thumbnail': r're:^https://iv\.okcdn\.ru/videoPreview\?',
},
}, {
# multiple ok_video
@@ -109,36 +132,41 @@ class BoostyIE(InfoExtractor):
'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
},
}],
'skip': 'post has been deleted',
}, {
# single external video (youtube)
'url': 'https://boosty.to/denischuzhoy/posts/6094a487-bcec-4cf8-a453-43313b463c38',
'url': 'https://boosty.to/futuremusicproduction/posts/32a8cae2-3252-49da-b285-0e014bc6e565',
'info_dict': {
'id': 'EXelTnve5lY',
'title': 'Послание Президента Федеральному Собранию | Класс народа',
'upload_date': '20210425',
'channel': 'Денис Чужой',
'tags': 'count:10',
'id': '-37FW_YQ3B4',
'title': 'Afro | Deep House FREE FLP',
'media_type': 'video',
'upload_date': '20250829',
'timestamp': 1756466005,
'channel': 'Future Music Production',
'tags': 'count:0',
'like_count': int,
'ext': 'mp4',
'duration': 816,
'ext': 'm4a',
'duration': 170,
'view_count': int,
'thumbnail': r're:^https://i\.ytimg\.com/',
'age_limit': 0,
'availability': 'public',
'categories': list,
'channel_follower_count': int,
'channel_id': 'UCCzVNbWZfYpBfyofCCUD_0w',
'channel_is_verified': bool,
'channel_id': 'UCKVYrFBYmci1e-T8NeHw2qg',
'channel_url': r're:^https://www\.youtube\.com/',
'comment_count': int,
'description': str,
'heatmap': 'count:100',
'live_status': str,
'playable_in_embed': bool,
'uploader': str,
'uploader_id': str,
'uploader_url': r're:^https://www\.youtube\.com/',
},
'expected_warnings': [
'Remote components challenge solver script',
'n challenge solving failed',
],
}]
_MP4_TYPES = ('tiny', 'lowest', 'low', 'medium', 'high', 'full_hd', 'quad_hd', 'ultra_hd')
@@ -207,13 +235,14 @@ class BoostyIE(InfoExtractor):
video_id = item.get('id') or post_id
entries.append({
'id': video_id,
'alt_title': post_title,
'formats': self._extract_formats(item.get('playerUrls'), video_id),
**common_metadata,
**traverse_obj(item, {
'title': ('title', {str}),
'duration': ('duration', {int_or_none}),
'view_count': ('viewsCounter', {int_or_none}),
'thumbnail': (('previewUrl', 'defaultPreview'), {url_or_none}),
'thumbnail': (('preview', 'defaultPreview'), {url_or_none}),
}, get_all=False)})
if not entries and not post.get('hasAccess'):
plugins/youtube_download/yt_dlp/extractor/cbc.py
+175 -41
View File
@@ -10,6 +10,7 @@ from ..utils import (
ExtractorError,
float_or_none,
int_or_none,
join_nonempty,
js_to_json,
jwt_decode_hs256,
mimetype2ext,
@@ -25,6 +26,7 @@ from ..utils import (
url_basename,
url_or_none,
urlencode_postdata,
urljoin,
)
from ..utils.traversal import require, traverse_obj, trim_str
@@ -105,7 +107,7 @@ class CBCIE(InfoExtractor):
# multiple CBC.APP.Caffeine.initInstance(...)
'url': 'http://www.cbc.ca/news/canada/calgary/dog-indoor-exercise-winter-1.3928238',
'info_dict': {
'title': 'Keep Rover active during the deep freeze with doggie pushups and other fun indoor tasks', # FIXME: actual title includes " | CBC News"
'title': 'Keep Rover active during the deep freeze with doggie pushups and other fun indoor tasks',
'id': 'dog-indoor-exercise-winter-1.3928238',
'description': 'md5:c18552e41726ee95bd75210d1ca9194c',
},
@@ -134,6 +136,13 @@ class CBCIE(InfoExtractor):
title = (self._og_search_title(webpage, default=None)
or self._html_search_meta('twitter:title', webpage, 'title', default=None)
or self._html_extract_title(webpage))
title = self._search_regex(
r'^(?P<title>.+?)(?:\s*[|–-]\s*CBC.*)?$',
title, 'cleaned title', group='title', default=title)
data = self._search_json(
r'window\.__INITIAL_STATE__\s*=', webpage,
'initial state', display_id, default={}, transform_source=js_to_json)
entries = [
self._extract_player_init(player_init, display_id)
for player_init in re.findall(r'CBC\.APP\.Caffeine\.initInstance\(({.+?})\);', webpage)]
@@ -143,6 +152,11 @@ class CBCIE(InfoExtractor):
r'<div[^>]+\bid=["\']player-(\d+)',
r'guid["\']\s*:\s*["\'](\d+)'):
media_ids.extend(re.findall(media_id_re, webpage))
media_ids.extend(traverse_obj(data, (
'detail', 'content', 'body', ..., 'content',
lambda _, v: v['type'] == 'polopoly_media', 'content', 'sourceId', {str})))
if content_id := traverse_obj(data, ('app', 'contentId', {str})):
media_ids.append(content_id)
entries.extend([
self.url_result(f'cbcplayer:{media_id}', 'CBCPlayer', media_id)
for media_id in orderedSet(media_ids)])
@@ -268,7 +282,7 @@ class CBCPlayerIE(InfoExtractor):
'duration': 2692.833,
'subtitles': {
'en-US': [{
'name': 'English Captions',
'name': r're:English',
'url': 'https://cbchls.akamaized.net/delivery/news-shows/2024/06/17/NAT_JUN16-00-55-00/NAT_JUN16_cc.vtt',
}],
},
@@ -322,6 +336,7 @@ class CBCPlayerIE(InfoExtractor):
'categories': ['Olympics Summer Soccer', 'Summer Olympics Replays', 'Summer Olympics Soccer Replays'],
'location': 'Canada',
},
'skip': 'Video no longer available',
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://www.cbc.ca/player/play/video/9.6459530',
@@ -380,7 +395,8 @@ class CBCPlayerIE(InfoExtractor):
video_id = self._match_id(url)
webpage = self._download_webpage(f'https://www.cbc.ca/player/play/{video_id}', video_id)
data = self._search_json(
r'window\.__INITIAL_STATE__\s*=', webpage, 'initial state', video_id)['video']['currentClip']
r'window\.__INITIAL_STATE__\s*=', webpage,
'initial state', video_id, transform_source=js_to_json)['video']['currentClip']
assets = traverse_obj(
data, ('media', 'assets', lambda _, v: url_or_none(v['key']) and v['type']))
@@ -492,12 +508,14 @@ class CBCPlayerPlaylistIE(InfoExtractor):
'info_dict': {
'id': 'news/tv shows/the national/latest broadcast',
},
'skip': 'Playlist no longer available',
}, {
'url': 'https://www.cbc.ca/player/news/Canada/North',
'playlist_mincount': 25,
'info_dict': {
'id': 'news/canada/north',
},
'skip': 'Playlist no longer available',
}]
def _real_extract(self, url):
@@ -524,6 +542,32 @@ class CBCGemBaseIE(InfoExtractor):
f'https://services.radio-canada.ca/ott/catalog/v2/gem/show/{item_id}',
display_id or item_id, query={'device': 'web'})
def _call_media_api(self, media_id, app_code='gem', display_id=None, headers=None):
media_data = self._download_json(
'https://services.radio-canada.ca/media/validation/v2/',
display_id or media_id, headers=headers, query={
'appCode': app_code,
'connectionType': 'hd',
'deviceType': 'ipad',
'multibitrate': 'true',
'output': 'json',
'tech': 'hls',
'manifestVersion': '2',
'manifestType': 'desktop',
'idMedia': media_id,
})
error_code = traverse_obj(media_data, ('errorCode', {int}))
if error_code == 1:
self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
if error_code == 35:
self.raise_login_required(method='password')
if error_code != 0:
error_message = join_nonempty(error_code, media_data.get('message'), delim=' - ')
raise ExtractorError(f'{self.IE_NAME} said: {error_message}')
return media_data
def _extract_item_info(self, item_info):
episode_number = None
title = traverse_obj(item_info, ('title', {str}))
@@ -551,7 +595,7 @@ class CBCGemBaseIE(InfoExtractor):
class CBCGemIE(CBCGemBaseIE):
IE_NAME = 'gem.cbc.ca'
_VALID_URL = r'https?://gem\.cbc\.ca/(?:media/)?(?P<id>[0-9a-z-]+/s(?P<season>[0-9]+)[a-z][0-9]+)'
_VALID_URL = r'https?://gem\.cbc\.ca/(?:media/)?(?P<id>[0-9a-z-]+/s(?P<season>[0-9]+)[a-z][0-9]{2,4})/?(?:[?#]|$)'
_TESTS = [{
# This is a normal, public, TV show video
'url': 'https://gem.cbc.ca/media/schitts-creek/s06e01',
@@ -693,29 +737,10 @@ class CBCGemIE(CBCGemBaseIE):
if claims_token := self._fetch_claims_token():
headers['x-claims-token'] = claims_token
m3u8_info = self._download_json(
'https://services.radio-canada.ca/media/validation/v2/',
video_id, headers=headers, query={
'appCode': 'gem',
'connectionType': 'hd',
'deviceType': 'ipad',
'multibitrate': 'true',
'output': 'json',
'tech': 'hls',
'manifestVersion': '2',
'manifestType': 'desktop',
'idMedia': item_info['idMedia'],
})
if m3u8_info.get('errorCode') == 1:
self.raise_geo_restricted(countries=['CA'])
elif m3u8_info.get('errorCode') == 35:
self.raise_login_required(method='password')
elif m3u8_info.get('errorCode') != 0:
raise ExtractorError(f'{self.IE_NAME} said: {m3u8_info.get("errorCode")} - {m3u8_info.get("message")}')
m3u8_url = self._call_media_api(
item_info['idMedia'], display_id=video_id, headers=headers)['url']
formats = self._extract_m3u8_formats(
m3u8_info['url'], video_id, 'mp4', m3u8_id='hls', query={'manifestType': ''})
m3u8_url, video_id, 'mp4', m3u8_id='hls', query={'manifestType': ''})
self._remove_duplicate_formats(formats)
for fmt in formats:
@@ -785,7 +810,128 @@ class CBCGemPlaylistIE(CBCGemBaseIE):
}), series=traverse_obj(show_info, ('title', {str})))
class CBCGemLiveIE(InfoExtractor):
class CBCGemContentIE(CBCGemBaseIE):
IE_NAME = 'gem.cbc.ca:content'
IE_DESC = False # Do not list
_VALID_URL = r'https?://gem\.cbc\.ca/(?P<id>[0-9a-z-]+)/?(?:[?#]|$)'
_TESTS = [{
# Series URL; content_type == 'Season'
'url': 'https://gem.cbc.ca/the-tunnel',
'playlist_count': 3,
'info_dict': {
'id': 'the-tunnel',
},
}, {
# Miniseries URL; content_type == 'Parts'
'url': 'https://gem.cbc.ca/summit-72',
'playlist_count': 1,
'info_dict': {
'id': 'summit-72',
},
}, {
# Olympics URL; content_type == 'Standalone'
'url': 'https://gem.cbc.ca/ski-jumping-nh-individual-womens-final-30086',
'info_dict': {
'id': 'ski-jumping-nh-individual-womens-final-30086',
'ext': 'mp4',
'title': 'Ski Jumping: NH Individual (Women\'s) - Final',
'description': 'md5:411c07c8a9a4a36344530b0c726bf8ab',
'duration': 12793,
'thumbnail': r're:https://[^.]+\.cbc\.ca/.+\.jpg',
'release_timestamp': 1770482100,
'release_date': '20260207',
'live_status': 'was_live',
},
}, {
# Movie URL; content_type == 'Standalone'; requires authentication
'url': 'https://gem.cbc.ca/copa-71',
'only_matching': True,
}]
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
data = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['data']
content_type = data['contentType']
self.write_debug(f'Routing for content type "{content_type}"')
if content_type == 'Standalone':
new_url = traverse_obj(data, (
'header', 'cta', 'media', 'url', {urljoin('https://gem.cbc.ca/')}))
if CBCGemOlympicsIE.suitable(new_url):
return self.url_result(new_url, CBCGemOlympicsIE)
# Manually construct non-Olympics standalone URLs to avoid returning trailer URLs
return self.url_result(f'https://gem.cbc.ca/{display_id}/s01e01', CBCGemIE)
# Handle series URLs (content_type == 'Season') and miniseries URLs (content_type == 'Parts')
def entries():
for playlist_url in traverse_obj(data, (
'content', ..., 'lineups', ..., 'url', {urljoin('https://gem.cbc.ca/')},
{lambda x: x if CBCGemPlaylistIE.suitable(x) else None},
)):
yield self.url_result(playlist_url, CBCGemPlaylistIE)
return self.playlist_result(entries(), display_id)
class CBCGemOlympicsIE(CBCGemBaseIE):
IE_NAME = 'gem.cbc.ca:olympics'
_VALID_URL = r'https?://gem\.cbc\.ca/(?P<id>(?:[0-9a-z]+-)+[0-9]{5,})/s01e(?P<media_id>[0-9]{5,})'
_TESTS = [{
'url': 'https://gem.cbc.ca/ski-jumping-nh-individual-womens-final-30086/s01e30086',
'info_dict': {
'id': 'ski-jumping-nh-individual-womens-final-30086',
'ext': 'mp4',
'title': 'Ski Jumping: NH Individual (Women\'s) - Final',
'description': 'md5:411c07c8a9a4a36344530b0c726bf8ab',
'duration': 12793,
'thumbnail': r're:https://[^.]+\.cbc\.ca/.+\.jpg',
'release_timestamp': 1770482100,
'release_date': '20260207',
'live_status': 'was_live',
},
}]
def _real_extract(self, url):
video_id, media_id = self._match_valid_url(url).group('id', 'media_id')
video_info = self._call_show_api(video_id)
item_info = traverse_obj(video_info, (
'content', ..., 'lineups', ..., 'items',
lambda _, v: v['formattedIdMedia'] == media_id, any, {require('item info')}))
live_status = {
'LiveEvent': 'is_live',
'Replay': 'was_live',
}.get(item_info.get('type'))
release_timestamp = traverse_obj(item_info, (
'metadata', (('live', 'startDate'), ('replay', 'airDate')), {parse_iso8601}, any))
if live_status == 'is_live' and release_timestamp and release_timestamp > time.time():
formats = []
live_status = 'is_upcoming'
self.raise_no_formats('This livestream has not yet started', expected=True)
else:
m3u8_url = self._call_media_api(media_id, 'medianetlive', video_id)['url']
formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', live=live_status == 'is_live')
return {
'id': video_id,
'formats': formats,
'live_status': live_status,
'release_timestamp': release_timestamp,
**traverse_obj(item_info, {
'title': ('title', {str}),
'description': ('description', {str}),
'thumbnail': ('images', 'card', 'url', {url_or_none}),
'duration': ('metadata', 'replay', 'duration', {int_or_none}),
}),
}
class CBCGemLiveIE(CBCGemBaseIE):
IE_NAME = 'gem.cbc.ca:live'
_VALID_URL = r'https?://gem\.cbc\.ca/live(?:-event)?/(?P<id>\d+)'
_TESTS = [
@@ -855,7 +1001,6 @@ class CBCGemLiveIE(InfoExtractor):
'only_matching': True,
},
]
_GEO_COUNTRIES = ['CA']
def _real_extract(self, url):
video_id = self._match_id(url)
@@ -884,19 +1029,8 @@ class CBCGemLiveIE(InfoExtractor):
live_status = 'is_upcoming'
self.raise_no_formats('This livestream has not yet started', expected=True)
else:
stream_data = self._download_json(
'https://services.radio-canada.ca/media/validation/v2/', video_id, query={
'appCode': 'medianetlive',
'connectionType': 'hd',
'deviceType': 'ipad',
'idMedia': video_stream_id,
'multibitrate': 'true',
'output': 'json',
'tech': 'hls',
'manifestType': 'desktop',
})
formats = self._extract_m3u8_formats(
stream_data['url'], video_id, 'mp4', live=live_status == 'is_live')
m3u8_url = self._call_media_api(video_stream_id, 'medianetlive', video_id)['url']
formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', live=live_status == 'is_live')
return {
'id': video_id,
plugins/youtube_download/yt_dlp/extractor/ccc.py
+20 -2
View File
@@ -18,23 +18,41 @@ class CCCIE(InfoExtractor):
'id': '1839',
'ext': 'mp4',
'title': 'Introduction to Processor Design',
'creator': 'byterazor',
'creators': ['byterazor'],
'description': 'md5:df55f6d073d4ceae55aae6f2fd98a0ac',
'thumbnail': r're:^https?://.*\.jpg$',
'upload_date': '20131228',
'timestamp': 1388188800,
'duration': 3710,
'tags': list,
'display_id': '30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor',
'view_count': int,
},
}, {
'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download',
'only_matching': True,
}, {
'url': 'https://media.ccc.de/v/39c3-schlechte-karten-it-sicherheit-im-jahr-null-der-epa-fur-alle',
'info_dict': {
'id': '16261',
'ext': 'mp4',
'title': 'Schlechte Karten - IT-Sicherheit im Jahr null der ePA für alle',
'display_id': '39c3-schlechte-karten-it-sicherheit-im-jahr-null-der-epa-fur-alle',
'description': 'md5:719a5a9a52630249d606219c55056cbf',
'view_count': int,
'duration': 3619,
'thumbnail': 'https://static.media.ccc.de/media/congress/2025/2403-2b5a6a8e-327e-594d-8f92-b91201d18a02.jpg',
'tags': list,
'creators': ['Bianca Kastl'],
'timestamp': 1767024900,
'upload_date': '20251229',
},
}]
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
event_id = self._search_regex(r"data-id='(\d+)'", webpage, 'event id')
event_id = self._search_regex(r"data-id=(['\"])(?P<event_id>\d+)\1", webpage, 'event id', group='event_id')
event_data = self._download_json(f'https://media.ccc.de/public/events/{event_id}', event_id)
formats = []
plugins/youtube_download/yt_dlp/extractor/common.py
+11 -5
View File
@@ -3,9 +3,7 @@ import collections
import contextlib
import functools
import getpass
import http.client
import http.cookiejar
import http.cookies
import inspect
import itertools
import json
@@ -661,9 +659,11 @@ class InfoExtractor:
if not self._ready:
self._initialize_pre_login()
if self.supports_login():
username, password = self._get_login_info()
if username:
self._perform_login(username, password)
# try login only if it would actually do anything
if type(self)._perform_login is not InfoExtractor._perform_login:
username, password = self._get_login_info()
if username:
self._perform_login(username, password)
elif self.get_param('username') and False not in (self.IE_DESC, self._NETRC_MACHINE):
self.report_warning(f'Login with password is not supported for this website. {self._login_hint("cookies")}')
self._real_initialize()
@@ -1385,6 +1385,11 @@ class InfoExtractor:
def _get_netrc_login_info(self, netrc_machine=None):
netrc_machine = netrc_machine or self._NETRC_MACHINE
if not netrc_machine:
raise ExtractorError(f'Missing netrc_machine and {type(self).__name__}._NETRC_MACHINE')
ALLOWED = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_'
if netrc_machine.startswith(('-', '_')) or not all(c in ALLOWED for c in netrc_machine):
raise ExtractorError(f'Invalid netrc machine: {netrc_machine!r}', expected=True)
cmd = self.get_param('netrc_cmd')
if cmd:
@@ -1883,6 +1888,7 @@ class InfoExtractor:
'ShallowReactive': indirect_reviver,
'Ref': indirect_reviver,
'Reactive': indirect_reviver,
'skipHydrate': indirect_reviver,
})
while True:
plugins/youtube_download/yt_dlp/extractor/dailymotion.py
+57 -2
View File
@@ -1,5 +1,6 @@
import functools
import json
import random
import re
import urllib.parse
@@ -363,6 +364,55 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
continue
yield update_url(player_url, query=query_string)
@staticmethod
def _generate_blockbuster_headers():
"""Randomize our HTTP header fingerprint to bust the HTTP Error 403 block"""
def random_letters(minimum, maximum):
# Omit vowels so we don't generate valid header names like 'authorization', etc
return ''.join(random.choices('bcdfghjklmnpqrstvwxz', k=random.randint(minimum, maximum)))
return {
random_letters(8, 24): random_letters(16, 32)
for _ in range(random.randint(2, 8))
}
def _extract_dailymotion_m3u8_formats_and_subtitles(self, media_url, video_id, live=False):
"""See https://github.com/yt-dlp/yt-dlp/issues/15526"""
ERROR_NOTE = 'Unable to download m3u8 information'
last_error = None
for note, kwargs in (
('Downloading m3u8 information with randomized headers', {
'headers': self._generate_blockbuster_headers(),
}),
('Retrying m3u8 download with Chrome impersonation', {
'impersonate': 'chrome',
'require_impersonation': True,
}),
('Retrying m3u8 download with Firefox impersonation', {
'impersonate': 'firefox',
'require_impersonation': True,
}),
):
try:
m3u8_doc = self._download_webpage(media_url, video_id, note, ERROR_NOTE, **kwargs)
break
except ExtractorError as e:
last_error = e.orig_msg
self.write_debug(f'{video_id}: {last_error}')
else:
if 'impersonation' not in last_error:
self.report_warning(last_error, video_id=video_id)
last_error = None
return [], {}, last_error
formats, subtitles = self._parse_m3u8_formats_and_subtitles(
m3u8_doc, media_url, 'mp4', m3u8_id='hls', live=live, fatal=False)
return formats, subtitles, last_error
def _real_extract(self, url):
url, smuggled_data = unsmuggle_url(url)
video_id, is_playlist, playlist_id = self._match_valid_url(url).group('id', 'is_playlist', 'playlist_id')
@@ -416,6 +466,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
is_live = media.get('isOnAir')
formats = []
subtitles = {}
expected_error = None
for quality, media_list in metadata['qualities'].items():
for m in media_list:
@@ -424,8 +475,8 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
if not media_url or media_type == 'application/vnd.lumberjack.manifest':
continue
if media_type == 'application/x-mpegURL':
fmt, subs = self._extract_m3u8_formats_and_subtitles(
media_url, video_id, 'mp4', live=is_live, m3u8_id='hls', fatal=False)
fmt, subs, expected_error = self._extract_dailymotion_m3u8_formats_and_subtitles(
media_url, video_id, live=is_live)
formats.extend(fmt)
self._merge_subtitles(subs, target=subtitles)
else:
@@ -442,6 +493,10 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
'width': width,
})
formats.append(f)
if not formats and expected_error:
self.raise_no_formats(expected_error, expected=True)
for f in formats:
f['url'] = f['url'].split('#')[0]
if not f.get('fps') and f['format_id'].endswith('@60'):
plugins/youtube_download/yt_dlp/extractor/err.py
+68
View File
@@ -2,6 +2,7 @@ from .common import InfoExtractor
from ..utils import (
clean_html,
int_or_none,
parse_iso8601,
str_or_none,
url_or_none,
)
@@ -222,3 +223,70 @@ class ERRJupiterIE(InfoExtractor):
'episode_id': ('id', {str_or_none}),
}) if data.get('type') == 'episode' else {}),
}
class ERRArhiivIE(InfoExtractor):
_VALID_URL = r'https://arhiiv\.err\.ee/video/(?:vaata/)?(?P<id>[^/?#]+)'
_TESTS = [{
'url': 'https://arhiiv.err.ee/video/kontsertpalad',
'info_dict': {
'id': 'kontsertpalad',
'ext': 'mp4',
'title': 'Kontsertpalad: 255 | L. Beethoveni sonaat c-moll, "Pateetiline"',
'description': 'md5:a70f4ff23c3618f3be63f704bccef063',
'series': 'Kontsertpalad',
'episode_id': 255,
'timestamp': 1666152162,
'upload_date': '20221019',
'release_year': 1970,
'modified_timestamp': 1718620982,
'modified_date': '20240617',
},
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://arhiiv.err.ee/video/vaata/koalitsioonileppe-allkirjastamine',
'info_dict': {
'id': 'koalitsioonileppe-allkirjastamine',
'ext': 'mp4',
'title': 'Koalitsioonileppe allkirjastamine',
'timestamp': 1710728222,
'upload_date': '20240318',
'release_timestamp': 1611532800,
'release_date': '20210125',
},
'params': {'skip_download': 'm3u8'},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
data = self._download_json(
f'https://arhiiv.err.ee/api/v1/content/video/{video_id}', video_id)
formats, subtitles = [], {}
if hls_url := traverse_obj(data, ('media', 'src', 'hls', {url_or_none})):
fmts, subs = self._extract_m3u8_formats_and_subtitles(
hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
if dash_url := traverse_obj(data, ('media', 'src', 'dash', {url_or_none})):
fmts, subs = self._extract_mpd_formats_and_subtitles(
dash_url, video_id, mpd_id='dash', fatal=False)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
return {
'id': video_id,
'formats': formats,
'subtitles': subtitles,
**traverse_obj(data, ('info', {
'title': ('title', {str}),
'series': ('seriesTitle', {str}, filter),
'series_id': ('seriesId', {str}, filter),
'episode_id': ('episode', {int_or_none}),
'description': ('synopsis', {str}, filter),
'timestamp': ('uploadDate', {parse_iso8601}),
'modified_timestamp': ('dateModified', {parse_iso8601}),
'release_timestamp': ('date', {parse_iso8601}),
'release_year': ('year', {int_or_none}),
})),
}
plugins/youtube_download/yt_dlp/extractor/extractors.py
+13 -7
View File
@@ -1,4 +1,4 @@
import inspect
import itertools
import os
from ..globals import LAZY_EXTRACTORS
@@ -17,12 +17,18 @@ else:
if not _CLASS_LOOKUP:
from . import _extractors
_CLASS_LOOKUP = {
name: value
for name, value in inspect.getmembers(_extractors)
if name.endswith('IE') and name != 'GenericIE'
}
_CLASS_LOOKUP['GenericIE'] = _extractors.GenericIE
members = tuple(
(name, getattr(_extractors, name))
for name in dir(_extractors)
if name.endswith('IE')
)
_CLASS_LOOKUP = dict(itertools.chain(
# Add Youtube first to improve matching performance
((name, value) for name, value in members if '.youtube' in value.__module__),
# Add Generic last so that it is the fallback
((name, value) for name, value in members if name != 'GenericIE'),
(('GenericIE', _extractors.GenericIE),),
))
# We want to append to the main lookup
_current = _extractors_context.value
plugins/youtube_download/yt_dlp/extractor/facebook.py
+76 -6
View File
@@ -4,6 +4,7 @@ import urllib.parse
from .common import InfoExtractor
from ..compat import compat_etree_fromstring
from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
clean_html,
@@ -1017,6 +1018,7 @@ class FacebookAdsIE(InfoExtractor):
'upload_date': '20240812',
'like_count': int,
},
'skip': 'Invalid URL',
}, {
'url': 'https://www.facebook.com/ads/library/?id=893637265423481',
'info_dict': {
@@ -1031,6 +1033,42 @@ class FacebookAdsIE(InfoExtractor):
},
'playlist_count': 3,
'skip': 'Invalid URL',
}, {
'url': 'https://www.facebook.com/ads/library/?id=312304267031140',
'info_dict': {
'id': '312304267031140',
'title': 'Casper Wave Hybrid Mattress',
'uploader': 'Casper',
'uploader_id': '224110981099062',
'uploader_url': 'https://www.facebook.com/Casper/',
'like_count': int,
},
'playlist_count': 2,
}, {
'url': 'https://www.facebook.com/ads/library/?id=874812092000430',
'info_dict': {
'id': '874812092000430',
'title': 'TikTok',
'uploader': 'Case \u00e0 Chocs',
'uploader_id': '112960472096793',
'uploader_url': 'https://www.facebook.com/Caseachocs/',
'like_count': int,
'description': 'md5:f02a255fcf7dce6ed40e9494cf4bc49a',
},
'playlist_count': 3,
}, {
'url': 'https://www.facebook.com/ads/library/?id=1704834754236452',
'info_dict': {
'id': '1704834754236452',
'ext': 'mp4',
'title': 'Get answers now!',
'description': 'Ask the best psychics and get accurate answers on questions that bother you!',
'uploader': 'Your Relationship Advisor',
'uploader_id': '108939234726306',
'uploader_url': 'https://www.facebook.com/100068970634636/',
'like_count': int,
'thumbnail': r're:https://.+/.+\.jpg',
},
}, {
'url': 'https://es-la.facebook.com/ads/library/?id=901230958115569',
'only_matching': True,
@@ -1060,15 +1098,45 @@ class FacebookAdsIE(InfoExtractor):
})
return formats
def _download_fb_webpage_and_verify(self, url, video_id):
# See https://github.com/yt-dlp/yt-dlp/issues/15577
try:
return self._download_webpage(url, video_id)
except ExtractorError as e:
if (
not isinstance(e.cause, HTTPError)
or e.cause.status != 403
or e.cause.reason != 'Client challenge'
):
raise
error_page = self._webpage_read_content(e.cause.response, url, video_id)
self.write_debug('Received a client challenge response')
challenge_path = self._search_regex(
r'fetch\s*\(\s*["\'](/__rd_verify[^"\']+)["\']',
error_page, 'challenge path')
# Successful response will set the necessary cookie
self._request_webpage(
urljoin(url, challenge_path), video_id, 'Requesting verification cookie',
'Unable to get verification cookie', data=b'')
return self._download_webpage(url, video_id)
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
webpage = self._download_fb_webpage_and_verify(url, video_id)
post_data = traverse_obj(
re.findall(r'data-sjs>({.*?ScheduledServerJS.*?})</script>', webpage), (..., {json.loads}))
data = get_first(post_data, (
'require', ..., ..., ..., '__bbox', 'require', ..., ..., ...,
'entryPointRoot', 'otherProps', 'deeplinkAdCard', 'snapshot', {dict}))
'require', ..., ..., ..., '__bbox', 'require', ..., ..., ..., (
('__bbox', 'result', 'data', 'ad_library_main', 'deeplink_ad_archive_result', 'deeplink_ad_archive'),
# old path
('entryPointRoot', 'otherProps', 'deeplinkAdCard'),
), 'snapshot', {dict}))
if not data:
raise ExtractorError('Unable to extract ad data')
@@ -1084,11 +1152,12 @@ class FacebookAdsIE(InfoExtractor):
'title': title,
'description': markup or None,
}, traverse_obj(data, {
'description': ('link_description', {lambda x: x if not x.startswith('{{product.') else None}),
'description': (
(('body', 'text'), 'link_description'),
{lambda x: x if not x.startswith('{{product.') else None}, any),
'uploader': ('page_name', {str}),
'uploader_id': ('page_id', {str_or_none}),
'uploader_url': ('page_profile_uri', {url_or_none}),
'timestamp': ('creation_time', {int_or_none}),
'like_count': ('page_like_count', {int_or_none}),
}))
@@ -1099,7 +1168,8 @@ class FacebookAdsIE(InfoExtractor):
entries.append({
'id': f'{video_id}_{idx}',
'title': entry.get('title') or title,
'description': traverse_obj(entry, 'body', 'link_description') or info_dict.get('description'),
'description': traverse_obj(
entry, 'body', 'link_description', expected_type=str) or info_dict.get('description'),
'thumbnail': url_or_none(entry.get('video_preview_image_url')),
'formats': self._extract_formats(entry),
})
plugins/youtube_download/yt_dlp/extractor/firsttv.py
+7
View File
@@ -3,10 +3,12 @@ import urllib.parse
from .common import InfoExtractor
from ..utils import (
determine_ext,
float_or_none,
int_or_none,
join_nonempty,
mimetype2ext,
parse_qs,
unescapeHTML,
unified_strdate,
url_or_none,
)
@@ -107,6 +109,11 @@ class FirstTVIE(InfoExtractor):
'timestamp': ('dvr_begin_at', {int_or_none}),
'upload_date': ('date_air', {unified_strdate}),
'duration': ('duration', {int_or_none}),
'chapters': ('episodes', lambda _, v: float_or_none(v['from']) is not None, {
'start_time': ('from', {float_or_none}),
'title': ('name', {str}, {unescapeHTML}),
'end_time': ('to', {float_or_none}),
}),
}),
'id': video_id,
'formats': formats,
plugins/youtube_download/yt_dlp/extractor/floatplane.py
+40 -34
View File
@@ -318,9 +318,48 @@ class FloatplaneIE(FloatplaneBaseIE):
self.raise_login_required()
class FloatplaneChannelIE(InfoExtractor):
class FloatplaneChannelBaseIE(InfoExtractor):
"""Subclasses must set _RESULT_IE, _BASE_URL and _PAGE_SIZE"""
def _fetch_page(self, display_id, creator_id, channel_id, page):
query = {
'id': creator_id,
'limit': self._PAGE_SIZE,
'fetchAfter': page * self._PAGE_SIZE,
}
if channel_id:
query['channel'] = channel_id
page_data = self._download_json(
f'{self._BASE_URL}/api/v3/content/creator', display_id,
query=query, note=f'Downloading page {page + 1}')
for post in page_data or []:
yield self.url_result(
f'{self._BASE_URL}/post/{post["id"]}',
self._RESULT_IE, id=post['id'], title=post.get('title'),
release_timestamp=parse_iso8601(post.get('releaseDate')))
def _real_extract(self, url):
creator, channel = self._match_valid_url(url).group('id', 'channel')
display_id = join_nonempty(creator, channel, delim='/')
creator_data = self._download_json(
f'{self._BASE_URL}/api/v3/creator/named',
display_id, query={'creatorURL[0]': creator})[0]
channel_data = traverse_obj(
creator_data, ('channels', lambda _, v: v['urlname'] == channel), get_all=False) or {}
return self.playlist_result(OnDemandPagedList(functools.partial(
self._fetch_page, display_id, creator_data['id'], channel_data.get('id')), self._PAGE_SIZE),
display_id, title=channel_data.get('title') or creator_data.get('title'),
description=channel_data.get('about') or creator_data.get('about'))
class FloatplaneChannelIE(FloatplaneChannelBaseIE):
_VALID_URL = r'https?://(?:(?:www|beta)\.)?floatplane\.com/channel/(?P<id>[\w-]+)/home(?:/(?P<channel>[\w-]+))?'
_BASE_URL = 'https://www.floatplane.com'
_PAGE_SIZE = 20
_RESULT_IE = FloatplaneIE
_TESTS = [{
'url': 'https://www.floatplane.com/channel/linustechtips/home/ltxexpo',
'info_dict': {
@@ -346,36 +385,3 @@ class FloatplaneChannelIE(InfoExtractor):
},
'playlist_mincount': 200,
}]
def _fetch_page(self, display_id, creator_id, channel_id, page):
query = {
'id': creator_id,
'limit': self._PAGE_SIZE,
'fetchAfter': page * self._PAGE_SIZE,
}
if channel_id:
query['channel'] = channel_id
page_data = self._download_json(
'https://www.floatplane.com/api/v3/content/creator', display_id,
query=query, note=f'Downloading page {page + 1}')
for post in page_data or []:
yield self.url_result(
f'https://www.floatplane.com/post/{post["id"]}',
FloatplaneIE, id=post['id'], title=post.get('title'),
release_timestamp=parse_iso8601(post.get('releaseDate')))
def _real_extract(self, url):
creator, channel = self._match_valid_url(url).group('id', 'channel')
display_id = join_nonempty(creator, channel, delim='/')
creator_data = self._download_json(
'https://www.floatplane.com/api/v3/creator/named',
display_id, query={'creatorURL[0]': creator})[0]
channel_data = traverse_obj(
creator_data, ('channels', lambda _, v: v['urlname'] == channel), get_all=False) or {}
return self.playlist_result(OnDemandPagedList(functools.partial(
self._fetch_page, display_id, creator_data['id'], channel_data.get('id')), self._PAGE_SIZE),
display_id, title=channel_data.get('title') or creator_data.get('title'),
description=channel_data.get('about') or creator_data.get('about'))
plugins/youtube_download/yt_dlp/extractor/francetv.py
+21 -5
View File
@@ -371,15 +371,16 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
class FranceTVInfoIE(FranceTVBaseInfoExtractor):
IE_NAME = 'francetvinfo.fr'
_VALID_URL = r'https?://(?:www|mobile|france3-regions)\.francetvinfo\.fr/(?:[^/]+/)*(?P<id>[^/?#&.]+)'
IE_NAME = 'franceinfo'
IE_DESC = 'franceinfo.fr (formerly francetvinfo.fr)'
_VALID_URL = r'https?://(?:www|mobile|france3-regions)\.france(?:tv)?info.fr/(?:[^/?#]+/)*(?P<id>[^/?#&.]+)'
_TESTS = [{
'url': 'https://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-jeudi-22-aout-2019_3561461.html',
'info_dict': {
'id': 'd12458ee-5062-48fe-bfdd-a30d6a01b793',
'ext': 'mp4',
'title': 'Soir 3',
'title': 'Soir 3 - Émission du jeudi 22 août 2019',
'upload_date': '20190822',
'timestamp': 1566510730,
'thumbnail': r're:^https?://.*\.jpe?g$',
@@ -398,7 +399,7 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor):
'info_dict': {
'id': '7d204c9e-a2d3-11eb-9e4c-000d3a23d482',
'ext': 'mp4',
'title': 'Covid-19 : une situation catastrophique à New Dehli - Édition du mercredi 21 avril 2021',
'title': 'Journal 20h00 - Covid-19 : une situation catastrophique à New Dehli',
'thumbnail': r're:^https?://.*\.jpe?g$',
'duration': 76,
'timestamp': 1619028518,
@@ -438,6 +439,18 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor):
'thumbnail': r're:https://[^/?#]+/v/[^/?#]+/x1080',
},
'add_ie': ['Dailymotion'],
'skip': 'Broken Dailymotion link',
}, {
'url': 'https://www.franceinfo.fr/monde/usa/presidentielle/donald-trump/etats-unis-un-risque-d-embrasement-apres-la-mort-d-un-manifestant_7764542.html',
'info_dict': {
'id': 'f920fcc2-fa20-11f0-ac98-57a09c50f7ce',
'ext': 'mp4',
'title': 'Affaires sensibles - Manifestant tué Le risque d\'embrasement',
'duration': 118,
'thumbnail': r're:https?://.+/.+\.jpg',
'timestamp': 1769367756,
'upload_date': '20260125',
},
}, {
'url': 'http://france3-regions.francetvinfo.fr/limousin/emissions/jt-1213-limousin',
'only_matching': True,
@@ -445,6 +458,9 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor):
# "<figure id=" pattern (#28792)
'url': 'https://www.francetvinfo.fr/culture/patrimoine/incendie-de-notre-dame-de-paris/notre-dame-de-paris-de-l-incendie-de-la-cathedrale-a-sa-reconstruction_4372291.html',
'only_matching': True,
}, {
'url': 'https://www.franceinfo.fr/replay-jt/france-2/20-heures/robert-de-niro-portrait-d-un-monument-du-cinema_7245456.html',
'only_matching': True,
}]
def _real_extract(self, url):
@@ -460,7 +476,7 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor):
video_id = (
traverse_obj(webpage, (
{find_element(tag='button', attr='data-cy', value='francetv-player-wrapper', html=True)},
{find_element(tag='(button|div)', attr='data-cy', value='francetv-player-wrapper', html=True, regex=True)},
{extract_attributes}, 'id'))
or self._search_regex(
(r'player\.load[^;]+src:\s*["\']([^"\']+)',
plugins/youtube_download/yt_dlp/extractor/frontro.py
-63
View File
@@ -99,66 +99,3 @@ class FrontroGroupBaseIE(FrontoBaseIE):
'modified_timestamp': ('updatedAt', {parse_iso8601}),
}),
}
class TheChosenIE(FrontroVideoBaseIE):
_CHANNEL_ID = '12884901895'
_VALID_URL = r'https?://(?:www\.)?watch\.thechosen\.tv/video/(?P<id>[0-9]+)'
_TESTS = [{
'url': 'https://watch.thechosen.tv/video/184683594325',
'md5': '3f878b689588c71b38ec9943c54ff5b0',
'info_dict': {
'id': '184683594325',
'ext': 'mp4',
'title': 'Season 3 Episode 2: Two by Two',
'description': 'md5:174c373756ecc8df46b403f4fcfbaf8c',
'comment_count': int,
'view_count': int,
'like_count': int,
'duration': 4212,
'thumbnail': r're:https://fastly\.frontrowcdn\.com/channels/12884901895/VIDEO_THUMBNAIL/184683594325/',
'timestamp': 1698954546,
'upload_date': '20231102',
'modified_timestamp': int,
'modified_date': str,
},
}, {
'url': 'https://watch.thechosen.tv/video/184683596189',
'md5': 'd581562f9d29ce82f5b7770415334151',
'info_dict': {
'id': '184683596189',
'ext': 'mp4',
'title': 'Season 4 Episode 8: Humble',
'description': 'md5:20a57bead43da1cf77cd5b0fe29bbc76',
'comment_count': int,
'view_count': int,
'like_count': int,
'duration': 5092,
'thumbnail': r're:https://fastly\.frontrowcdn\.com/channels/12884901895/VIDEO_THUMBNAIL/184683596189/',
'timestamp': 1715019474,
'upload_date': '20240506',
'modified_timestamp': int,
'modified_date': str,
},
}]
class TheChosenGroupIE(FrontroGroupBaseIE):
_CHANNEL_ID = '12884901895'
_VIDEO_EXTRACTOR = TheChosenIE
_VIDEO_URL_TMPL = 'https://watch.thechosen.tv/video/%s'
_VALID_URL = r'https?://(?:www\.)?watch\.thechosen\.tv/group/(?P<id>[0-9]+)'
_TESTS = [{
'url': 'https://watch.thechosen.tv/group/309237658592',
'info_dict': {
'id': '309237658592',
'title': 'Season 3',
'timestamp': 1746203969,
'upload_date': '20250502',
'modified_timestamp': int,
'modified_date': str,
},
'playlist_count': 8,
}]
plugins/youtube_download/yt_dlp/extractor/getcourseru.py
+1 -1
View File
@@ -59,7 +59,7 @@ class GetCourseRuIE(InfoExtractor):
'marafon.mani-beauty.com',
'on.psbook.ru',
]
_BASE_URL_RE = rf'https?://(?:(?!player02\.)[^.]+\.getcourse\.(?:ru|io)|{"|".join(map(re.escape, _DOMAINS))})'
_BASE_URL_RE = rf'https?://(?:(?!player02\.)[a-zA-Z0-9-]+\.getcourse\.(?:ru|io)|{"|".join(map(re.escape, _DOMAINS))})'
_VALID_URL = [
rf'{_BASE_URL_RE}/(?!pl/|teach/)(?P<id>[^?#]+)',
rf'{_BASE_URL_RE}/(?:pl/)?teach/control/lesson/view\?(?:[^#]+&)?id=(?P<id>\d+)',
plugins/youtube_download/yt_dlp/extractor/lbry.py
+1
View File
@@ -95,6 +95,7 @@ class LBRYBaseIE(InfoExtractor):
'_type': 'url',
'id': item['claim_id'],
'url': self._permanent_url(url, item['name'], item['claim_id']),
'ie_key': 'LBRY',
}
def _playlist_entries(self, url, display_id, claim_param, metadata):
plugins/youtube_download/yt_dlp/extractor/learningonscreen.py
+1 -1
View File
@@ -29,7 +29,7 @@ class LearningOnScreenIE(InfoExtractor):
}]
def _real_initialize(self):
if not self._get_cookies('https://learningonscreen.ac.uk/').get('PHPSESSID-BOB-LIVE'):
if not self._get_cookies('https://learningonscreen.ac.uk/').get('PHPSESSID-LOS-LIVE'):
self.raise_login_required(method='session_cookies')
def _real_extract(self, url):
plugins/youtube_download/yt_dlp/extractor/locipo.py
+209
View File
@@ -0,0 +1,209 @@
import functools
import math
from .streaks import StreaksBaseIE
from ..networking import HEADRequest
from ..utils import (
InAdvancePagedList,
clean_html,
js_to_json,
parse_iso8601,
parse_qs,
str_or_none,
)
from ..utils.traversal import require, traverse_obj
class LocipoBaseIE(StreaksBaseIE):
_API_BASE = 'https://web-api.locipo.jp'
_BASE_URL = 'https://locipo.jp'
_UUID_RE = r'[\da-f]{8}(?:-[\da-f]{4}){3}-[\da-f]{12}'
def _call_api(self, path, item_id, note, fatal=True):
return self._download_json(
f'{self._API_BASE}/{path}', item_id,
f'Downloading {note} API JSON',
f'Unable to download {note} API JSON',
fatal=fatal)
class LocipoIE(LocipoBaseIE):
_VALID_URL = [
fr'https?://locipo\.jp/creative/(?P<id>{LocipoBaseIE._UUID_RE})',
fr'https?://locipo\.jp/embed/?\?(?:[^#]+&)?id=(?P<id>{LocipoBaseIE._UUID_RE})',
]
_TESTS = [{
'url': 'https://locipo.jp/creative/fb5ffeaa-398d-45ce-bb49-0e221b5f94f1',
'info_dict': {
'id': 'fb5ffeaa-398d-45ce-bb49-0e221b5f94f1',
'ext': 'mp4',
'title': 'リアルカレカノ#4 ～伊達さゆりと勉強しよっ？～',
'description': 'md5:70a40c202f3fb7946b61e55fa015094c',
'display_id': '5a2947fe596441f5bab88a61b0432d0d',
'live_status': 'not_live',
'modified_date': r're:\d{8}',
'modified_timestamp': int,
'release_timestamp': 1711789200,
'release_date': '20240330',
'series': 'リアルカレカノ',
'series_id': '1142',
'tags': 'count:4',
'thumbnail': r're:https?://.+\.(?:jpg|png)',
'timestamp': 1756984919,
'upload_date': '20250904',
'uploader': '東海テレビ',
'uploader_id': 'locipo-prod',
},
}, {
'url': 'https://locipo.jp/embed/?id=71a334a0-2b25-406f-9d96-88f341f571c2',
'info_dict': {
'id': '71a334a0-2b25-406f-9d96-88f341f571c2',
'ext': 'mp4',
'title': '#1 オーディション／ゲスト伊藤美来、豊田萌絵',
'description': 'md5:5bbcf532474700439cf56ceb6a15630e',
'display_id': '0ab32634b884499a84adb25de844c551',
'live_status': 'not_live',
'modified_date': r're:\d{8}',
'modified_timestamp': int,
'release_timestamp': 1751623200,
'release_date': '20250704',
'series': '声優ラジオのウラカブリ～Locipo出張所～',
'series_id': '1454',
'tags': 'count:6',
'thumbnail': r're:https?://.+\.(?:jpg|png)',
'timestamp': 1757002966,
'upload_date': '20250904',
'uploader': 'テレビ愛知',
'uploader_id': 'locipo-prod',
},
}, {
'url': 'https://locipo.jp/creative/bff9950d-229b-4fe9-911a-7fa71a232f35?list=69a5b15c-901f-4828-a336-30c0de7612d3',
'info_dict': {
'id': '69a5b15c-901f-4828-a336-30c0de7612d3',
'title': '見て・乗って・語りたい。 東海の鉄道沼',
},
'playlist_mincount': 3,
}, {
'url': 'https://locipo.jp/creative/a0751a7f-c7dd-4a10-a7f1-e12720bdf16c?list=006cff3f-ba74-42f0-b4fd-241486ebda2b',
'info_dict': {
'id': 'a0751a7f-c7dd-4a10-a7f1-e12720bdf16c',
'ext': 'mp4',
'title': '#839 人間真空パック',
'description': 'md5:9fe190333b6975c5001c8c9cbe20d276',
'display_id': 'c2b4c9f4a6d648bd8e3c320e384b9d56',
'live_status': 'not_live',
'modified_date': r're:\d{8}',
'modified_timestamp': int,
'release_timestamp': 1746239400,
'release_date': '20250503',
'series': 'でんじろう先生のはぴエネ！',
'series_id': '202',
'tags': 'count:3',
'thumbnail': r're:https?://.+\.(?:jpg|png)',
'timestamp': 1756975909,
'upload_date': '20250904',
'uploader': '中京テレビ',
'uploader_id': 'locipo-prod',
},
'params': {'noplaylist': True},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
playlist_id = traverse_obj(parse_qs(url), ('list', -1, {str}))
if self._yes_playlist(playlist_id, video_id):
return self.url_result(
f'{self._BASE_URL}/playlist/{playlist_id}', LocipoPlaylistIE)
creatives = self._call_api(f'creatives/{video_id}', video_id, 'Creatives')
media_id = traverse_obj(creatives, ('media_id', {str}, {require('Streaks media ID')}))
webpage = self._download_webpage(url, video_id)
config = self._search_json(
r'window\.__NUXT__\.config\s*=', webpage, 'config', video_id, transform_source=js_to_json)
api_key = traverse_obj(config, ('public', 'streaksVodPlaybackApiKey', {str}, {require('api key')}))
return {
**self._extract_from_streaks_api('locipo-prod', media_id, headers={
'Origin': 'https://locipo.jp',
'X-Streaks-Api-Key': api_key,
}),
**traverse_obj(creatives, {
'title': ('name', {clean_html}),
'description': ('description', {clean_html}, filter),
'release_timestamp': ('publication_started_at', {parse_iso8601}),
'tags': ('keyword', {clean_html}, {lambda x: x.split(',')}, ..., {str.strip}, filter),
'uploader': ('company', 'name', {clean_html}, filter),
}),
**traverse_obj(creatives, ('series', {
'series': ('name', {clean_html}, filter),
'series_id': ('id', {str_or_none}),
})),
'id': video_id,
}
class LocipoPlaylistIE(LocipoBaseIE):
_VALID_URL = [
fr'https?://locipo\.jp/(?P<type>playlist)/(?P<id>{LocipoBaseIE._UUID_RE})',
r'https?://locipo\.jp/(?P<type>series)/(?P<id>\d+)',
]
_TESTS = [{
'url': 'https://locipo.jp/playlist/35d3dd2b-531d-4824-8575-b1c527d29538',
'info_dict': {
'id': '35d3dd2b-531d-4824-8575-b1c527d29538',
'title': 'レシピ集',
},
'playlist_mincount': 135,
}, {
# Redirects to https://locipo.jp/series/1363
'url': 'https://locipo.jp/playlist/fef7c4fb-741f-4d6a-a3a6-754f354302a2',
'info_dict': {
'id': '1363',
'title': 'CBCアナウンサー公式【みてちょてれび】',
'description': 'md5:50a1b23e63112d5c06c882835c8c1fb1',
},
'playlist_mincount': 38,
}, {
'url': 'https://locipo.jp/series/503',
'info_dict': {
'id': '503',
'title': 'FishingLover東海',
'description': '東海地区の釣り場でフィッシングの魅力を余すところなくご紹介！！',
},
'playlist_mincount': 223,
}]
_PAGE_SIZE = 100
def _fetch_page(self, path, playlist_id, page):
creatives = self._download_json(
f'{self._API_BASE}/{path}/{playlist_id}/creatives',
playlist_id, f'Downloading page {page + 1}', query={
'premium': False,
'live': False,
'limit': self._PAGE_SIZE,
'offset': page * self._PAGE_SIZE,
})
for video_id in traverse_obj(creatives, ('items', ..., 'id', {str})):
yield self.url_result(f'{self._BASE_URL}/creative/{video_id}', LocipoIE)
def _real_extract(self, url):
playlist_type, playlist_id = self._match_valid_url(url).group('type', 'id')
if urlh := self._request_webpage(HEADRequest(url), playlist_id, fatal=False):
playlist_type, playlist_id = self._match_valid_url(urlh.url).group('type', 'id')
path = 'playlists' if playlist_type == 'playlist' else 'series'
creatives = self._call_api(
f'{path}/{playlist_id}/creatives', playlist_id, path.capitalize())
entries = InAdvancePagedList(
functools.partial(self._fetch_page, path, playlist_id),
math.ceil(int(creatives['total']) / self._PAGE_SIZE), self._PAGE_SIZE)
return self.playlist_result(
entries, playlist_id,
**traverse_obj(creatives, ('items', ..., playlist_type, {
'title': ('name', {clean_html}, filter),
'description': ('description', {clean_html}, filter),
}, any)))
plugins/youtube_download/yt_dlp/extractor/matchitv.py
+38
View File
@@ -0,0 +1,38 @@
from .common import InfoExtractor
from ..utils import join_nonempty, unified_strdate
from ..utils.traversal import traverse_obj
class MatchiTVIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?matchi\.tv/watch/?\?(?:[^#]+&)?s=(?P<id>[0-9a-zA-Z]+)'
_TESTS = [{
'url': 'https://matchi.tv/watch?s=0euhjzrxsjm',
'info_dict': {
'id': '0euhjzrxsjm',
'ext': 'mp4',
'title': 'Court 2 at Stratford Padel Club 2024-07-13T18:32:24',
'thumbnail': 'https://thumbnails.padelgo.tv/0euhjzrxsjm.jpg',
'upload_date': '20240713',
},
}, {
'url': 'https://matchi.tv/watch?s=FkKDJ9SvAx1',
'only_matching': True,
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
loaded_media = traverse_obj(
self._search_nextjs_data(webpage, video_id, fatal=False),
('props', 'pageProps', 'loadedMedia', {dict})) or {}
start_date_time = traverse_obj(loaded_media, ('startDateTime', {str}))
return {
'id': video_id,
'title': join_nonempty(loaded_media.get('courtDescription'), start_date_time, delim=' '),
'thumbnail': f'https://thumbnails.padelgo.tv/{video_id}.jpg',
'upload_date': unified_strdate(start_date_time),
'formats': self._extract_m3u8_formats(
f'https://streams.padelgo.tv/v2/streams/m3u8/{video_id}/anonymous/playlist.m3u8',
video_id, 'mp4', m3u8_id='hls'),
}
plugins/youtube_download/yt_dlp/extractor/mixcloud.py
+1 -1
View File
@@ -25,7 +25,7 @@ class MixcloudBaseIE(InfoExtractor):
%s
}
}''' % (lookup_key, username, f', slug: "{slug}"' if slug else '', object_fields), # noqa: UP031
})['data'][lookup_key]
}, impersonate=True)['data'][lookup_key]
class MixcloudIE(MixcloudBaseIE):
plugins/youtube_download/yt_dlp/extractor/neteasemusic.py
+32 -7
View File
@@ -156,18 +156,36 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
'id': '17241424',
'ext': 'mp3',
'title': 'Opus 28',
'upload_date': '20080211',
'timestamp': 1202745600,
'upload_date': '20060912',
'timestamp': 1158076800,
'duration': 263,
'thumbnail': r're:^http.*\.jpg',
'album': 'Piano Solos Vol. 2',
'album': 'Piano Solos, Vol. 2',
'album_artist': 'Dustin O\'Halloran',
'average_rating': int,
'description': '[00:05.00]纯音乐，请欣赏\n',
'description': 'md5:b566b92c55ca348df65d206c5d689576',
'album_artists': ['Dustin O\'Halloran'],
'creators': ['Dustin O\'Halloran'],
'subtitles': {'lyrics': [{'ext': 'lrc'}]},
},
}, {
'url': 'https://music.163.com/#/song?id=2755669231',
'info_dict': {
'id': '2755669231',
'ext': 'mp3',
'title': '十二月-Departure',
'upload_date': '20251111',
'timestamp': 1762876800,
'duration': 188,
'thumbnail': r're:^http.*\.jpg',
'album': '円',
'album_artist': 'ひとひら',
'average_rating': int,
'description': 'md5:deee249c8c9c3e2c54ecdab36e87d174',
'album_artists': ['ひとひら'],
'creators': ['ひとひら'],
'subtitles': {'lyrics': [{'ext': 'lrc', 'data': 'md5:d32b4425a5d6c9fa249ca6e803dd0401'}]},
},
}, {
'url': 'https://y.music.163.com/m/song?app_version=8.8.45&id=95670&uct2=sKnvS4+0YStsWkqsPhFijw%3D%3D&dlt=0846',
'md5': 'b896be78d8d34bd7bb665b26710913ff',
@@ -241,9 +259,16 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
'lyrics': [{'data': original, 'ext': 'lrc'}],
}
lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\])([^\n]+)'
original_ts_texts = re.findall(lyrics_expr, original)
translation_ts_dict = dict(re.findall(lyrics_expr, translated))
def collect_lyrics(lrc):
lyrics_expr = r'\[([0-9]{2}):([0-9]{2})[:.]([0-9]{2,})\]([^\n]+)'
matches = re.findall(lyrics_expr, lrc)
return (
(f'[{minute}:{sec}.{msec}]', text)
for minute, sec, msec, text in matches
)
original_ts_texts = collect_lyrics(original)
translation_ts_dict = dict(collect_lyrics(translated))
merged = '\n'.join(
join_nonempty(f'{timestamp}{text}', translation_ts_dict.get(timestamp, ''), delim=' / ')
plugins/youtube_download/yt_dlp/extractor/odnoklassniki.py
+2 -4
View File
@@ -9,13 +9,13 @@ from ..utils import (
int_or_none,
qualities,
smuggle_url,
traverse_obj,
unescapeHTML,
unified_strdate,
unsmuggle_url,
url_or_none,
urlencode_postdata,
)
from ..utils.traversal import find_element, traverse_obj
class OdnoklassnikiIE(InfoExtractor):
@@ -264,9 +264,7 @@ class OdnoklassnikiIE(InfoExtractor):
note='Downloading desktop webpage',
headers={'Referer': smuggled['referrer']} if smuggled.get('referrer') else {})
error = self._search_regex(
r'[^>]+class="vp_video_stub_txt"[^>]*>([^<]+)<',
webpage, 'error', default=None)
error = traverse_obj(webpage, {find_element(cls='vp_video_stub_txt')})
# Direct link from boosty
if (error == 'The author of this video has not been found or is blocked'
and not smuggled.get('referrer') and mode == 'videoembed'):
plugins/youtube_download/yt_dlp/extractor/opencast.py
+9 -4
View File
@@ -33,7 +33,8 @@ class OpencastBaseIE(InfoExtractor):
vid\.igb\.illinois\.edu|
cursosabertos\.c3sl\.ufpr\.br|
mcmedia\.missioncollege\.org|
clases\.odon\.edu\.uy
clases\.odon\.edu\.uy|
oc-p\.uni-jena\.de
)'''
_UUID_RE = r'[\da-fA-F]{8}-[\da-fA-F]{4}-[\da-fA-F]{4}-[\da-fA-F]{4}-[\da-fA-F]{12}'
@@ -106,7 +107,7 @@ class OpencastBaseIE(InfoExtractor):
class OpencastIE(OpencastBaseIE):
_VALID_URL = rf'''(?x)
https?://(?P<host>{OpencastBaseIE._INSTANCES_RE})/paella/ui/watch\.html\?
https?://(?P<host>{OpencastBaseIE._INSTANCES_RE})/paella[0-9]*/ui/watch\.html\?
(?:[^#]+&)?id=(?P<id>{OpencastBaseIE._UUID_RE})'''
_API_BASE = 'https://%s/search/episode.json?id=%s'
@@ -131,8 +132,12 @@ class OpencastIE(OpencastBaseIE):
def _real_extract(self, url):
host, video_id = self._match_valid_url(url).group('host', 'id')
return self._parse_mediapackage(
self._call_api(host, video_id)['search-results']['result']['mediapackage'])
response = self._call_api(host, video_id)
package = traverse_obj(response, (
('search-results', 'result'),
('result', ...), # Path needed for oc-p.uni-jena.de
'mediapackage', {dict}, any)) or {}
return self._parse_mediapackage(package)
class OpencastPlaylistIE(OpencastBaseIE):
plugins/youtube_download/yt_dlp/extractor/patreon.py
+133 -48
View File
@@ -1,6 +1,5 @@
import functools
import itertools
import urllib.parse
from .common import InfoExtractor
from .sproutvideo import VidsIoIE
@@ -11,30 +10,38 @@ from ..utils import (
ExtractorError,
clean_html,
determine_ext,
extract_attributes,
float_or_none,
int_or_none,
mimetype2ext,
parse_iso8601,
smuggle_url,
str_or_none,
update_url_query,
url_or_none,
urljoin,
)
from ..utils.traversal import require, traverse_obj, value
from ..utils.traversal import (
find_elements,
require,
traverse_obj,
trim_str,
value,
)
class PatreonBaseIE(InfoExtractor):
@functools.cached_property
def patreon_user_agent(self):
# Patreon mobile UA is needed to avoid triggering Cloudflare anti-bot protection.
# Newer UA yields higher res m3u8 formats for locked posts, but gives 401 if not logged-in
# Patreon mobile UA yields higher res m3u8 for locked posts, but gives 401 if not logged-in
if self._get_cookies('https://www.patreon.com/').get('session_id'):
return 'Patreon/72.2.28 (Android; Android 14; Scale/2.10)'
return 'Patreon/7.6.28 (Android; Android 11; Scale/2.10)'
return 'Patreon/126.9.0.15 (Android; Android 14; Scale/2.10)'
return None
def _call_api(self, ep, item_id, query=None, headers=None, fatal=True, note=None):
if headers is None:
headers = {}
if 'User-Agent' not in headers:
if 'User-Agent' not in headers and self.patreon_user_agent:
headers['User-Agent'] = self.patreon_user_agent
if query:
query.update({'json-api-version': 1.0})
@@ -43,7 +50,9 @@ class PatreonBaseIE(InfoExtractor):
return self._download_json(
f'https://www.patreon.com/api/{ep}',
item_id, note=note if note else 'Downloading API JSON',
query=query, fatal=fatal, headers=headers)
query=query, fatal=fatal, headers=headers,
# If not using Patreon mobile UA, we need impersonation due to Cloudflare
impersonate=not self.patreon_user_agent)
except ExtractorError as e:
if not isinstance(e.cause, HTTPError) or mimetype2ext(e.cause.response.headers.get('Content-Type')) != 'json':
raise
@@ -121,6 +130,7 @@ class PatreonIE(PatreonBaseIE):
'channel_is_verified': True,
'chapters': 'count:4',
'timestamp': 1423689666,
'media_type': 'video',
},
'params': {
'noplaylist': True,
@@ -161,7 +171,7 @@ class PatreonIE(PatreonBaseIE):
'uploader_url': 'https://www.patreon.com/loish',
'description': 'md5:e2693e97ee299c8ece47ffdb67e7d9d2',
'title': 'VIDEO // sketchbook flipthrough',
'uploader': 'Loish ',
'uploader': 'Loish',
'tags': ['sketchbook', 'video'],
'channel_id': '1641751',
'channel_url': 'https://www.patreon.com/loish',
@@ -274,8 +284,73 @@ class PatreonIE(PatreonBaseIE):
'channel_id': '9346307',
},
'params': {'getcomments': True},
}, {
# Inlined media in post; uses _extract_from_media_api
'url': 'https://www.patreon.com/posts/scottfalco-146966245',
'info_dict': {
'id': '146966245',
'ext': 'mp4',
'title': 'scottfalco 1080',
'description': 'md5:a3f29bbd0a46b4821ec3400957c98aa2',
'uploader': 'Insanimate',
'uploader_id': '2828146',
'uploader_url': 'https://www.patreon.com/Insanimate',
'channel_id': '6260877',
'channel_url': 'https://www.patreon.com/Insanimate',
'channel_follower_count': int,
'comment_count': int,
'like_count': int,
'duration': 7.833333,
'timestamp': 1767061800,
'upload_date': '20251230',
},
}]
_RETURN_TYPE = 'video'
_HTTP_HEADERS = {
# Must be all-lowercase 'referer' so we can smuggle it to Generic, SproutVideo, and Vimeo.
# patreon.com URLs redirect to www.patreon.com; this matters when requesting mux.com m3u8s
'referer': 'https://www.patreon.com/',
}
def _extract_from_media_api(self, media_id):
attributes = traverse_obj(
self._call_api(f'media/{media_id}', media_id, fatal=False),
('data', 'attributes', {dict}))
if not attributes:
return None
info_dict = traverse_obj(attributes, {
'title': ('file_name', {lambda x: x.rpartition('.')[0]}),
'timestamp': ('created_at', {parse_iso8601}),
'duration': ('display', 'duration', {float_or_none}),
})
info_dict['id'] = media_id
playback_url = traverse_obj(
attributes, ('display', (None, 'viewer_playback_data'), 'url', {url_or_none}, any))
download_url = traverse_obj(attributes, ('download_url', {url_or_none}))
if playback_url and mimetype2ext(attributes.get('mimetype')) == 'm3u8':
info_dict['formats'], info_dict['subtitles'] = self._extract_m3u8_formats_and_subtitles(
playback_url, media_id, 'mp4', fatal=False, headers=self._HTTP_HEADERS)
for f in info_dict['formats']:
f['http_headers'] = self._HTTP_HEADERS
if transcript_url := traverse_obj(attributes, ('display', 'transcript_url', {url_or_none})):
info_dict['subtitles'].setdefault('en', []).append({
'url': transcript_url,
'ext': 'vtt',
})
elif playback_url or download_url:
info_dict['formats'] = [{
# If playback_url is available, download_url is a duplicate lower resolution format
'url': playback_url or download_url,
'vcodec': 'none' if attributes.get('media_type') != 'video' else None,
}]
if not info_dict.get('formats'):
return None
return info_dict
def _real_extract(self, url):
video_id = self._match_id(url)
@@ -299,6 +374,7 @@ class PatreonIE(PatreonBaseIE):
'comment_count': ('comment_count', {int_or_none}),
})
seen_media_ids = set()
entries = []
idx = 0
for include in traverse_obj(post, ('included', lambda _, v: v['type'])):
@@ -320,6 +396,8 @@ class PatreonIE(PatreonBaseIE):
'url': download_url,
'alt_title': traverse_obj(media_attributes, ('file_name', {str})),
})
if media_id := traverse_obj(include, ('id', {str})):
seen_media_ids.add(media_id)
elif include_type == 'user':
info.update(traverse_obj(include, {
@@ -340,34 +418,29 @@ class PatreonIE(PatreonBaseIE):
'channel_follower_count': ('attributes', 'patron_count', {int_or_none}),
}))
# Must be all-lowercase 'referer' so we can smuggle it to Generic, SproutVideo, and Vimeo.
# patreon.com URLs redirect to www.patreon.com; this matters when requesting mux.com m3u8s
headers = {'referer': 'https://www.patreon.com/'}
if embed_url := traverse_obj(attributes, ('embed', 'url', {url_or_none})):
# Convert useless vimeo.com URLs to useful player.vimeo.com embed URLs
vimeo_id, vimeo_hash = self._search_regex(
r'//vimeo\.com/(\d+)(?:/([\da-f]+))?', embed_url,
'vimeo id', group=(1, 2), default=(None, None))
if vimeo_id:
embed_url = update_url_query(
f'https://player.vimeo.com/video/{vimeo_id}',
{'h': vimeo_hash or []})
if VimeoIE.suitable(embed_url):
entry = self.url_result(
VimeoIE._smuggle_referrer(embed_url, self._HTTP_HEADERS['referer']),
VimeoIE, url_transparent=True)
else:
entry = self.url_result(smuggle_url(embed_url, self._HTTP_HEADERS))
# handle Vimeo embeds
if traverse_obj(attributes, ('embed', 'provider')) == 'Vimeo':
v_url = urllib.parse.unquote(self._html_search_regex(
r'(https(?:%3A%2F%2F|://)player\.vimeo\.com.+app_id(?:=|%3D)+\d+)',
traverse_obj(attributes, ('embed', 'html', {str})), 'vimeo url', fatal=False) or '')
if url_or_none(v_url) and self._request_webpage(
v_url, video_id, 'Checking Vimeo embed URL', headers=headers,
fatal=False, errnote=False, expected_status=429): # 429 is TLS fingerprint rejection
entries.append(self.url_result(
VimeoIE._smuggle_referrer(v_url, headers['referer']),
VimeoIE, url_transparent=True))
embed_url = traverse_obj(attributes, ('embed', 'url', {url_or_none}))
if embed_url and (urlh := self._request_webpage(
embed_url, video_id, 'Checking embed URL', headers=headers,
fatal=False, errnote=False, expected_status=403)):
# Vimeo's Cloudflare anti-bot protection will return HTTP status 200 for 404, so we need
# to check for "Sorry, we couldn&amp;rsquo;t find that page" in the meta description tag
meta_description = clean_html(self._html_search_meta(
'description', self._webpage_read_content(urlh, embed_url, video_id, fatal=False), default=None))
# Password-protected vids.io embeds return 403 errors w/o --video-password or session cookie
if ((urlh.status != 403 and meta_description != 'Sorry, we couldn’t find that page')
or VidsIoIE.suitable(embed_url)):
entries.append(self.url_result(smuggle_url(embed_url, headers)))
if urlh := self._request_webpage(
embed_url, video_id, 'Checking embed URL', headers=self._HTTP_HEADERS,
fatal=False, errnote=False, expected_status=(403, 429), # Ignore Vimeo 429's
):
# Password-protected vids.io embeds return 403 errors w/o --video-password or session cookie
if VidsIoIE.suitable(embed_url) or urlh.status != 403:
entries.append(entry)
post_file = traverse_obj(attributes, ('post_file', {dict}))
if post_file:
@@ -381,13 +454,27 @@ class PatreonIE(PatreonBaseIE):
})
elif name == 'video' or determine_ext(post_file.get('url')) == 'm3u8':
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
post_file['url'], video_id, headers=headers)
post_file['url'], video_id, headers=self._HTTP_HEADERS)
for f in formats:
f['http_headers'] = self._HTTP_HEADERS
entries.append({
'id': video_id,
'formats': formats,
'subtitles': subtitles,
'http_headers': headers,
})
if media_id := traverse_obj(post_file, ('media_id', {int}, {str_or_none})):
seen_media_ids.add(media_id)
for media_id in traverse_obj(attributes, (
'content', {find_elements(attr='data-media-id', value=r'\d+', regex=True, html=True)},
..., {extract_attributes}, 'data-media-id',
)):
# Inlined media may be duplicates of what was extracted above
if media_id in seen_media_ids:
continue
if media := self._extract_from_media_api(media_id):
entries.append(media)
seen_media_ids.add(media_id)
can_view_post = traverse_obj(attributes, 'current_user_can_view')
comments = None
@@ -538,14 +625,13 @@ class PatreonCampaignIE(PatreonBaseIE):
'info_dict': {
'id': '9631148',
'title': 'Anything Else?',
'description': 'md5:2ee1db4aed2f9460c2b295825a24aa08',
'description': 'md5:b2f20eec4cb5520d9a4be4971f28add5',
'uploader': 'dan ',
'uploader_id': '13852412',
'uploader_url': 'https://www.patreon.com/anythingelse',
'channel': 'Anything Else?',
'channel_id': '9631148',
'channel_url': 'https://www.patreon.com/anythingelse',
'channel_follower_count': int,
'age_limit': 0,
'thumbnail': r're:https?://.+/.+',
},
@@ -590,16 +676,15 @@ class PatreonCampaignIE(PatreonBaseIE):
break
def _real_extract(self, url):
campaign_id, vanity = self._match_valid_url(url).group('campaign_id', 'vanity')
if campaign_id is None:
webpage = self._download_webpage(url, vanity, headers={'User-Agent': self.patreon_user_agent})
campaign_id = traverse_obj(self._search_nextjs_data(webpage, vanity, default=None), (
'props', 'pageProps', 'bootstrapEnvelope', 'pageBootstrap', 'campaign', 'data', 'id', {str}))
if not campaign_id:
campaign_id = traverse_obj(self._search_nextjs_v13_data(webpage, vanity), (
((..., 'value', 'campaign', 'data'), lambda _, v: v['type'] == 'campaign'),
'id', {str}, any, {require('campaign ID')}))
results = self._call_api('search', vanity, query={
'q': vanity,
'page[size]': '5',
})['data']
campaign_id = traverse_obj(results, (
lambda _, v: v['type'] == 'campaign-document' and v['attributes']['url'].lower().endswith(f'/{vanity.lower()}'),
'id', {trim_str(start='campaign_')}, filter, any, {require('campaign ID')}))
params = {
'json-api-use-default-includes': 'false',
plugins/youtube_download/yt_dlp/extractor/pbs.py
+18
View File
@@ -453,6 +453,23 @@ class PBSIE(InfoExtractor):
'url': 'https://player.pbs.org/portalplayer/3004638221/?uid=',
'only_matching': True,
},
{
# Next.js v13+, see https://github.com/yt-dlp/yt-dlp/issues/13299
'url': 'https://www.pbs.org/video/caregiving',
'info_dict': {
'id': '3101776876',
'ext': 'mp4',
'title': 'Caregiving - Caregiving',
'description': 'A documentary revealing America’s caregiving crisis through intimate stories and expert insight.',
'display_id': 'caregiving',
'duration': 6783,
'thumbnail': 'https://image.pbs.org/video-assets/BSrSkcc-asset-mezzanine-16x9-nlcxQts.jpg',
'chapters': [],
},
'params': {
'skip_download': True,
},
},
]
_ERRORS = {
101: 'We\'re sorry, but this video is not yet available.',
@@ -506,6 +523,7 @@ class PBSIE(InfoExtractor):
r"(?s)window\.PBS\.playerConfig\s*=\s*{.*?id\s*:\s*'([0-9]+)',",
r'<div[^>]+\bdata-cove-id=["\'](\d+)"', # http://www.pbs.org/wgbh/roadshow/watch/episode/2105-indianapolis-hour-2/
r'<iframe[^>]+\bsrc=["\'](?:https?:)?//video\.pbs\.org/widget/partnerplayer/(\d+)', # https://www.pbs.org/wgbh/masterpiece/episodes/victoria-s2-e1/
r'\\"videoTPMediaId\\":\\\"(\d+)\\"', # Next.js v13, e.g. https://www.pbs.org/video/caregiving
r'\bhttps?://player\.pbs\.org/[\w-]+player/(\d+)', # last pattern to avoid false positives
]
plugins/youtube_download/yt_dlp/extractor/peertube.py
+1 -1
View File
@@ -1322,7 +1322,7 @@ class PeerTubeIE(InfoExtractor):
)
(?P<id>{_UUID_RE})
'''
_EMBED_REGEX = [r'''(?x)<iframe[^>]+\bsrc=["\'](?P<url>(?:https?:)?//{_INSTANCES_RE}/videos/embed/{cls._UUID_RE})''']
_EMBED_REGEX = [rf'''(?x)<iframe[^>]+\bsrc=["\'](?P<url>(?:https?:)?//{_INSTANCES_RE}/videos/embed/{_UUID_RE})''']
_TESTS = [{
'url': 'https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d',
'md5': '8563064d245a4be5705bddb22bb00a28',
plugins/youtube_download/yt_dlp/extractor/pornhub.py
+3 -2
View File
@@ -128,7 +128,7 @@ class PornHubIE(PornHubBaseIE):
_VALID_URL = rf'''(?x)
https?://
(?:
(?:[^/]+\.)?
(?:[a-zA-Z0-9.-]+\.)?
{PornHubBaseIE._PORNHUB_HOST_RE}
/(?:(?:view_video\.php|video/show)\?viewkey=|embed/)|
(?:www\.)?thumbzilla\.com/video/
@@ -506,6 +506,7 @@ class PornHubIE(PornHubBaseIE):
'cast': ({find_elements(attr='data-label', value='pornstar')}, ..., {clean_html}),
}),
'subtitles': subtitles,
'http_headers': {'Referer': f'https://www.{host}/'},
}, info)
@@ -533,7 +534,7 @@ class PornHubPlaylistBaseIE(PornHubBaseIE):
class PornHubUserIE(PornHubPlaylistBaseIE):
_VALID_URL = rf'(?P<url>https?://(?:[^/]+\.)?{PornHubBaseIE._PORNHUB_HOST_RE}/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/?#&]+))(?:[?#&]|/(?!videos)|$)'
_VALID_URL = rf'(?P<url>https?://(?:[a-zA-Z0-9.-]+\.)?{PornHubBaseIE._PORNHUB_HOST_RE}/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/?#&]+))(?:[?#&]|/(?!videos)|$)'
_TESTS = [{
'url': 'https://www.pornhub.com/model/zoe_ph',
'playlist_mincount': 118,
plugins/youtube_download/yt_dlp/extractor/rtp.py
+169 -33
View File
@@ -8,6 +8,7 @@ from ..utils import (
determine_ext,
int_or_none,
js_to_json,
make_archive_id,
parse_duration,
parse_iso8601,
url_or_none,
@@ -16,12 +17,12 @@ from ..utils.traversal import traverse_obj
class RTPIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?rtp\.pt/play/(?:[^/#?]+/)?p(?P<program_id>\d+)/(?P<id>e\d+)'
_VALID_URL = r'https?://(?:www\.)?rtp\.pt/play/(?:[^/#?]+/)?(?P<program_id>p\d+)/(?P<episode_id>e\d+)(?:/[^/#?]+/(?P<asset_id>\d+))?'
_TESTS = [{
'url': 'http://www.rtp.pt/play/p405/e174042/paixoes-cruzadas',
'md5': 'e736ce0c665e459ddb818546220b4ef8',
'info_dict': {
'id': 'e174042',
'id': '395769',
'ext': 'mp3',
'title': 'Paixões Cruzadas',
'description': 'md5:af979e58ba0ab73f78435fc943fdb070',
@@ -32,12 +33,15 @@ class RTPIE(InfoExtractor):
'modified_date': '20190327',
'timestamp': 1417219200,
'upload_date': '20141129',
'episode_id': 'e174042',
'series_id': 'p405',
'_old_archive_ids': ['rtp e174042'],
},
}, {
'url': 'https://www.rtp.pt/play/zigzag/p13166/e757904/25-curiosidades-25-de-abril',
'md5': '5b4859940e3adef61247a77dfb76046a',
'info_dict': {
'id': 'e757904',
'id': '1226642',
'ext': 'mp4',
'title': 'Estudar ou não estudar',
'description': 'md5:3bfd7eb8bebfd5711a08df69c9c14c35',
@@ -50,13 +54,16 @@ class RTPIE(InfoExtractor):
'episode_number': 2,
'episode': 'Estudar ou não estudar',
'modified_date': '20240404',
'episode_id': 'e757904',
'series_id': 'p13166',
'_old_archive_ids': ['rtp e757904'],
},
}, {
# Episode not accessible through API
'url': 'https://www.rtp.pt/play/estudoemcasa/p7776/e500050/portugues-1-ano',
'md5': '57660c0b46db9f22118c52cbd65975e4',
'info_dict': {
'id': 'e500050',
'id': '871639',
'ext': 'mp4',
'title': 'Português - 1.º ano',
'duration': 1669.0,
@@ -64,6 +71,67 @@ class RTPIE(InfoExtractor):
'upload_date': '20201020',
'timestamp': 1603180799,
'thumbnail': 'https://cdn-images.rtp.pt/EPG/imagens/39482_59449_64850.png?v=3&w=860',
'episode_id': 'e500050',
'series_id': 'p7776',
'_old_archive_ids': ['rtp e500050'],
},
'expected_warnings': ['Episode data not found in API response; falling back to web extraction'],
}, {
# Ambiguous URL for 1st part of a multi-part episode without --no-playlist
'url': 'https://www.rtp.pt/play/p14335/e877072/a-nossa-tarde',
'info_dict': {
'id': 'e877072',
'title': 'A Nossa Tarde',
'duration': 6545.0,
'series': 'A Nossa Tarde',
'series_id': 'p14335',
'season': '2025',
'episode_id': 'e877072',
'timestamp': 1758560188,
'upload_date': '20250922',
'modified_timestamp': 1758563110,
'modified_date': '20250922',
},
'playlist_count': 3,
}, {
# Ambiguous URL for 1st part of a multi-part episode with --no-playlist
'url': 'https://www.rtp.pt/play/p14335/e877072/a-nossa-tarde',
'md5': '2aa3c89c95e852d6f04168b95d0d0632',
'info_dict': {
'id': '1364711',
'ext': 'mp4',
'title': 'A Nossa Tarde',
'duration': 1292.0,
'thumbnail': r're:https://cdn-images\.rtp\.pt/multimedia/screenshots/p14335/p14335_1_20250922155118e161t0312\.jpg',
'series': 'A Nossa Tarde',
'series_id': 'p14335',
'season': '2025',
'episode_id': 'e877072',
'timestamp': 1758560188,
'upload_date': '20250922',
'modified_timestamp': 1758563110,
'modified_date': '20250922',
'_old_archive_ids': ['rtp e877072'],
},
'params': {'noplaylist': True},
}, {
# Unambiguous URL for 2nd part of a multi-part episode
'url': 'https://www.rtp.pt/play/p14335/e877072/a-nossa-tarde/1364744',
'md5': 'b624767af558a557372a6fcd1dcdfa17',
'info_dict': {
'id': '1364744',
'ext': 'mp4',
'title': 'A Nossa Tarde',
'duration': 3270.0,
'thumbnail': r're:https://cdn-images\.rtp\.pt/multimedia/screenshots/p14335/p14335_2_20250922165718e161t0412\.jpg',
'series': 'A Nossa Tarde',
'series_id': 'p14335',
'season': '2025',
'episode_id': 'e877072',
'timestamp': 1758560188,
'upload_date': '20250922',
'modified_timestamp': 1758563110,
'modified_date': '20250922',
},
}]
@@ -92,19 +160,19 @@ class RTPIE(InfoExtractor):
return None
return url.replace('/drm-fps/', '/hls/').replace('/drm-dash/', '/dash/')
def _extract_formats(self, media_urls, episode_id):
def _extract_formats(self, media_urls, display_id):
formats = []
subtitles = {}
for media_url in set(traverse_obj(media_urls, (..., {url_or_none}, {self._cleanup_media_url}))):
ext = determine_ext(media_url)
if ext == 'm3u8':
fmts, subs = self._extract_m3u8_formats_and_subtitles(
media_url, episode_id, m3u8_id='hls', fatal=False)
media_url, display_id, m3u8_id='hls', fatal=False)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
elif ext == 'mpd':
fmts, subs = self._extract_mpd_formats_and_subtitles(
media_url, episode_id, mpd_id='dash', fatal=False)
media_url, display_id, mpd_id='dash', fatal=False)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
else:
@@ -114,24 +182,12 @@ class RTPIE(InfoExtractor):
})
return formats, subtitles
def _extract_from_api(self, program_id, episode_id):
auth_token = self._fetch_auth_token()
if not auth_token:
return
episode_data = traverse_obj(self._download_json(
f'https://www.rtp.pt/play/api/1/get-episode/{program_id}/{episode_id[1:]}', episode_id,
query={'include_assets': 'true', 'include_webparams': 'true'},
headers={
'Accept': '*/*',
'Authorization': f'Bearer {auth_token}',
'User-Agent': self._USER_AGENT,
}, fatal=False), 'result', {dict})
if not episode_data:
return
asset_urls = traverse_obj(episode_data, ('assets', 0, 'asset_url', {dict}))
def _extract_asset(self, asset_data, episode_id, episode_info, archive_compat=False):
asset_id = asset_data['asset_id']
asset_urls = traverse_obj(asset_data, ('asset_url', {dict}))
media_urls = traverse_obj(asset_urls, (
((('hls', 'dash'), 'stream_url'), ('multibitrate', ('url_hls', 'url_dash'))),))
formats, subtitles = self._extract_formats(media_urls, episode_id)
formats, subtitles = self._extract_formats(media_urls, asset_id)
for sub_data in traverse_obj(asset_urls, ('subtitles', 'vtt_list', lambda _, v: url_or_none(v['file']))):
subtitles.setdefault(sub_data.get('code') or 'pt', []).append({
@@ -140,17 +196,63 @@ class RTPIE(InfoExtractor):
})
return {
'id': episode_id,
**episode_info,
'id': asset_id,
'episode_id': episode_id,
# asset_id is a unique identifier for all RTP videos, while episode_id is duplicated
# across all parts of a multi-part episode. Older versions of this IE returned
# episode_id as the video id and would only download the first part of multi-part eps.
# For download archive compat, we should return the episode_id as the old archive id
# *only* when extracting single-part episodes OR the *first* part of a multi-part ep.
'_old_archive_ids': [make_archive_id(self, episode_id)] if archive_compat else None,
'formats': formats,
'subtitles': subtitles,
'thumbnail': traverse_obj(episode_data, ('assets', 0, 'asset_thumbnail', {url_or_none})),
**traverse_obj(asset_data, {
'thumbnail': ('asset_thumbnail', {url_or_none}),
'duration': ('asset_duration', {parse_duration}),
'webpage_url': ('web', 'url', {url_or_none}),
}),
}
def _report_fallback_warning(self, missing_info_name='required info', display_id=None):
self.report_warning(
f'{missing_info_name.capitalize()} not found in API response; falling back to web extraction',
video_id=display_id)
def _entries(self, assets, episode_id, episode_info):
# Only pass archive_compat=True for the first entry without an asset_id in its webpage_url
for idx, asset_data in enumerate(assets):
yield self._extract_asset(asset_data, episode_id, episode_info, archive_compat=not idx)
def _extract_from_api(self, program_id, episode_id, asset_id):
auth_token = self._fetch_auth_token()
if not auth_token:
self._report_fallback_warning('auth token', episode_id)
return None
episode_data = traverse_obj(self._download_json(
f'https://www.rtp.pt/play/api/1/get-episode/{program_id[1:]}/{episode_id[1:]}',
asset_id or episode_id, query={'include_assets': 'true', 'include_webparams': 'true'},
headers={
'Accept': '*/*',
'Authorization': f'Bearer {auth_token}',
'User-Agent': self._USER_AGENT,
}, fatal=False), 'result', {dict})
if not episode_data:
self._report_fallback_warning('episode data', episode_id)
return None
episode_info = {
'id': episode_id, # playlist id
'episode_id': episode_id,
'series_id': program_id,
**traverse_obj(episode_data, ('episode', {
'title': (('episode_title', 'program_title'), {str}, filter, any),
'alt_title': ('episode_subtitle', {str}, filter),
'description': (('episode_description', 'episode_summary'), {str}, filter, any),
'timestamp': ('episode_air_date', {parse_iso8601(delimiter=' ')}),
'modified_timestamp': ('episode_lastchanged', {parse_iso8601(delimiter=' ')}),
'duration': ('episode_duration_complete', {parse_duration}),
'duration': ('episode_duration_complete', {parse_duration}), # playlist duration
'episode': ('episode_title', {str}, filter),
'episode_number': ('episode_number', {int_or_none}),
'season': ('program_season', {str}, filter),
@@ -158,6 +260,30 @@ class RTPIE(InfoExtractor):
})),
}
assets = traverse_obj(episode_data, ('assets', lambda _, v: v['asset_id']))
if not assets:
self._report_fallback_warning('asset IDs', episode_id)
return None
if asset_id:
asset_data = traverse_obj(assets, (lambda _, v: v['asset_id'] == asset_id, any))
if not asset_data:
self._report_fallback_warning(f'asset {asset_id}', episode_id)
return None
return self._extract_asset(asset_data, episode_id, episode_info)
asset_data = assets[0]
if self._yes_playlist(
len(assets) > 1 and episode_id, asset_data['asset_id'],
playlist_label='multi-part episode', video_label='individual part',
):
return self.playlist_result(
self._entries(assets, episode_id, episode_info), **episode_info)
# Pass archive_compat=True so we return _old_archive_ids for URLs without an asset_id
return self._extract_asset(asset_data, episode_id, episode_info, archive_compat=True)
_RX_OBFUSCATION = re.compile(r'''(?xs)
atob\s*\(\s*decodeURIComponent\s*\(\s*
(\[[0-9A-Za-z%,'"]*\])
@@ -172,25 +298,35 @@ class RTPIE(InfoExtractor):
)).decode('iso-8859-1')),
data)
def _extract_from_html(self, url, episode_id):
webpage = self._download_webpage(url, episode_id)
def _extract_from_html(self, url, program_id, episode_id, asset_id):
webpage = self._download_webpage(url, asset_id or episode_id)
if not asset_id:
asset_id = self._search_regex(r'\basset_id\s*:\s*"(\d+)"', webpage, 'asset ID')
old_archive_ids = [make_archive_id(self, episode_id)]
else:
old_archive_ids = None
formats = []
subtitles = {}
media_urls = traverse_obj(re.findall(r'(?:var\s+f\s*=|RTPPlayer\({[^}]+file:)\s*({[^}]+}|"[^"]+")', webpage), (
-1, (({self.__unobfuscate}, {js_to_json}, {json.loads}, {dict.values}, ...), {json.loads})))
formats, subtitles = self._extract_formats(media_urls, episode_id)
formats, subtitles = self._extract_formats(media_urls, asset_id)
return {
'id': episode_id,
'id': asset_id,
'episode_id': episode_id,
'series_id': program_id,
'formats': formats,
'subtitles': subtitles,
'description': self._html_search_meta(['og:description', 'twitter:description'], webpage, default=None),
'thumbnail': self._html_search_meta(['og:image', 'twitter:image'], webpage, default=None),
**self._search_json_ld(webpage, episode_id, default={}),
**self._search_json_ld(webpage, asset_id, default={}),
'title': self._html_search_meta(['og:title', 'twitter:title'], webpage, default=None),
'_old_archive_ids': old_archive_ids,
}
def _real_extract(self, url):
program_id, episode_id = self._match_valid_url(url).group('program_id', 'id')
return self._extract_from_api(program_id, episode_id) or self._extract_from_html(url, episode_id)
program_id, episode_id, asset_id = self._match_valid_url(url).group('program_id', 'episode_id', 'asset_id')
return (
self._extract_from_api(program_id, episode_id, asset_id)
or self._extract_from_html(url, program_id, episode_id, asset_id))
plugins/youtube_download/yt_dlp/extractor/rumble.py
+1 -1
View File
@@ -405,7 +405,7 @@ class RumbleChannelIE(InfoExtractor):
for video_url in traverse_obj(
get_elements_html_by_class('videostream__link', webpage), (..., {extract_attributes}, 'href'),
):
yield self.url_result(urljoin('https://rumble.com', video_url))
yield self.url_result(urljoin('https://rumble.com', video_url), RumbleIE)
def _real_extract(self, url):
url, playlist_id = self._match_valid_url(url).groups()
plugins/youtube_download/yt_dlp/extractor/sauceplus.py
+17 -1
View File
@@ -1,4 +1,4 @@
from .floatplane import FloatplaneBaseIE
from .floatplane import FloatplaneBaseIE, FloatplaneChannelBaseIE
class SaucePlusIE(FloatplaneBaseIE):
@@ -39,3 +39,19 @@ class SaucePlusIE(FloatplaneBaseIE):
def _real_initialize(self):
if not self._get_cookies(self._BASE_URL).get('__Host-sp-sess'):
self.raise_login_required()
class SaucePlusChannelIE(FloatplaneChannelBaseIE):
_VALID_URL = r'https?://(?:(?:www|beta)\.)?sauceplus\.com/channel/(?P<id>[\w-]+)/home(?:/(?P<channel>[\w-]+))?'
_BASE_URL = 'https://www.sauceplus.com'
_RESULT_IE = SaucePlusIE
_PAGE_SIZE = 20
_TESTS = [{
'url': 'https://www.sauceplus.com/channel/williamosman/home',
'info_dict': {
'id': 'williamosman',
'title': 'William Osman',
'description': 'md5:a67bc961d23c293b2c5308d84f34f26c',
},
'playlist_mincount': 158,
}]
plugins/youtube_download/yt_dlp/extractor/sbs.py
+2 -2
View File
@@ -146,8 +146,8 @@ class SBSIE(InfoExtractor):
'release_year': ('releaseYear', {int_or_none}),
'duration': ('duration', ({float_or_none}, {parse_duration})),
'is_live': ('liveStream', {bool}),
'age_limit': (('classificationID', 'contentRating'), {str.upper}, {
lambda x: self._AUS_TV_PARENTAL_GUIDELINES.get(x)}), # dict.get is unhashable in py3.7
'age_limit': (
('classificationID', 'contentRating'), {str.upper}, {self._AUS_TV_PARENTAL_GUIDELINES.get}),
}, get_all=False),
**traverse_obj(media, {
'categories': (('genres', ...), ('taxonomy', ('genre', 'subgenre'), 'name'), {str}),
plugins/youtube_download/yt_dlp/extractor/soundcloud.py
+46 -9
View File
@@ -6,6 +6,7 @@ import re
from .common import InfoExtractor, SearchInfoExtractor
from ..networking import HEADRequest
from ..networking.exceptions import HTTPError
from ..networking.impersonate import ImpersonateTarget
from ..utils import (
ExtractorError,
float_or_none,
@@ -118,9 +119,9 @@ class SoundcloudBaseIE(InfoExtractor):
self.cache.store('soundcloud', 'client_id', client_id)
def _update_client_id(self):
webpage = self._download_webpage('https://soundcloud.com/', None)
webpage = self._download_webpage('https://soundcloud.com/', None, 'Downloading main page')
for src in reversed(re.findall(r'<script[^>]+src="([^"]+)"', webpage)):
script = self._download_webpage(src, None, fatal=False)
script = self._download_webpage(src, None, 'Downloading JS asset', fatal=False)
if script:
client_id = self._search_regex(
r'client_id\s*:\s*"([0-9a-zA-Z]{32})"',
@@ -136,13 +137,13 @@ class SoundcloudBaseIE(InfoExtractor):
if non_fatal:
del kwargs['fatal']
query = kwargs.get('query', {}).copy()
for _ in range(2):
for is_first_attempt in (True, False):
query['client_id'] = self._CLIENT_ID
kwargs['query'] = query
try:
return self._download_json(*args, **kwargs)
except ExtractorError as e:
if isinstance(e.cause, HTTPError) and e.cause.status in (401, 403):
if is_first_attempt and isinstance(e.cause, HTTPError) and e.cause.status in (401, 403):
self._store_client_id(None)
self._update_client_id()
continue
@@ -152,7 +153,10 @@ class SoundcloudBaseIE(InfoExtractor):
raise
def _initialize_pre_login(self):
self._CLIENT_ID = self.cache.load('soundcloud', 'client_id') or 'a3e059563d7fd3372b49b37f00a00bcf'
self._CLIENT_ID = self.cache.load('soundcloud', 'client_id')
if self._CLIENT_ID:
return
self._update_client_id()
def _verify_oauth_token(self, token):
if self._request_webpage(
@@ -317,9 +321,16 @@ class SoundcloudBaseIE(InfoExtractor):
continue
# XXX: if not extract_flat, 429 error must be caught where _extract_info_dict is called
stream_url = traverse_obj(self._call_api(
format_url, track_id, f'Downloading {short_identifier} format info JSON',
query=query, headers=self._HEADERS), ('url', {url_or_none}))
try:
stream_url = traverse_obj(self._call_api(
format_url, track_id, f'Downloading {short_identifier} format info JSON',
query=query, headers=self._HEADERS), ('url', {url_or_none}))
except ExtractorError as e:
if isinstance(e.cause, HTTPError) and e.cause.status == 404:
self.report_warning(f'{short_identifier} format not found', video_id=track_id)
continue
raise
if invalid_url(stream_url):
continue
format_urls.add(stream_url)
@@ -830,6 +841,30 @@ class SoundcloudPagedPlaylistBaseIE(SoundcloudBaseIE):
'entries': self._entries(base_url, playlist_id),
}
@functools.cached_property
def _browser_impersonate_target(self):
available_targets = self._downloader._get_available_impersonate_targets()
if not available_targets:
# impersonate=True gives a generic warning when no impersonation targets are available
return True
# Any browser target older than chrome-116 is 403'd by Datadome
MIN_SUPPORTED_TARGET = ImpersonateTarget('chrome', '116', 'windows', '10')
version_as_float = lambda x: float(x.version) if x.version else 0
# Always try to use the newest Chrome target available
filtered = sorted([
target[0] for target in available_targets
if target[0].client == 'chrome' and target[0].os in ('windows', 'macos')
], key=version_as_float)
if not filtered or version_as_float(filtered[-1]) < version_as_float(MIN_SUPPORTED_TARGET):
# All available targets are inadequate or newest available Chrome target is too old, so
# warn the user to upgrade their dependency to a version with the minimum supported target
return MIN_SUPPORTED_TARGET
return filtered[-1]
def _entries(self, url, playlist_id):
# Per the SoundCloud documentation, the maximum limit for a linked partitioning query is 200.
# https://developers.soundcloud.com/blog/offset-pagination-deprecated
@@ -844,7 +879,9 @@ class SoundcloudPagedPlaylistBaseIE(SoundcloudBaseIE):
try:
response = self._call_api(
url, playlist_id, query=query, headers=self._HEADERS,
note=f'Downloading track page {i + 1}')
note=f'Downloading track page {i + 1}',
# See: https://github.com/yt-dlp/yt-dlp/issues/15660
impersonate=self._browser_impersonate_target)
break
except ExtractorError as e:
# Downloading page may result in intermittent 502 HTTP error
plugins/youtube_download/yt_dlp/extractor/spankbang.py
+9 -7
View File
@@ -3,6 +3,7 @@ import re
from .common import InfoExtractor
from ..utils import (
ExtractorError,
clean_html,
determine_ext,
merge_dicts,
parse_duration,
@@ -12,6 +13,7 @@ from ..utils import (
urlencode_postdata,
urljoin,
)
from ..utils.traversal import find_element, traverse_obj, trim_str
class SpankBangIE(InfoExtractor):
@@ -122,7 +124,7 @@ class SpankBangIE(InfoExtractor):
}), headers={
'Referer': url,
'X-Requested-With': 'XMLHttpRequest',
})
}, impersonate=True)
for format_id, format_url in stream.items():
if format_url and isinstance(format_url, list):
@@ -178,9 +180,9 @@ class SpankBangPlaylistIE(InfoExtractor):
def _real_extract(self, url):
mobj = self._match_valid_url(url)
playlist_id = mobj.group('id')
webpage = self._download_webpage(
url, playlist_id, headers={'Cookie': 'country=US; mobile=on'})
country = self.get_param('geo_bypass_country') or 'US'
self._set_cookie('.spankbang.com', 'country', country.upper())
webpage = self._download_webpage(url, playlist_id, impersonate=True)
entries = [self.url_result(
urljoin(url, mobj.group('path')),
@@ -189,8 +191,8 @@ class SpankBangPlaylistIE(InfoExtractor):
r'<a[^>]+\bhref=(["\'])(?P<path>/?[\da-z]+-(?P<id>[\da-z]+)/playlist/[^"\'](?:(?!\1).)*)\1',
webpage)]
title = self._html_search_regex(
r'<em>([^<]+)</em>\s+playlist\s*<', webpage, 'playlist title',
fatal=False)
title = traverse_obj(webpage, (
{find_element(tag='h1', attr='data-testid', value='playlist-title')},
{clean_html}, {trim_str(end=' Playlist')}))
return self.playlist_result(entries, playlist_id, title)
plugins/youtube_download/yt_dlp/extractor/steam.py
+43 -32
View File
@@ -8,15 +8,12 @@ from ..utils import (
extract_attributes,
join_nonempty,
js_to_json,
parse_resolution,
str_or_none,
url_basename,
url_or_none,
)
from ..utils.traversal import (
find_element,
find_elements,
traverse_obj,
trim_str,
)
from ..utils.traversal import find_element, traverse_obj
class SteamIE(InfoExtractor):
@@ -27,7 +24,7 @@ class SteamIE(InfoExtractor):
'id': '105600',
'title': 'Terraria',
},
'playlist_mincount': 3,
'playlist_mincount': 5,
}, {
'url': 'https://store.steampowered.com/app/271590/Grand_Theft_Auto_V/',
'info_dict': {
@@ -37,6 +34,39 @@ class SteamIE(InfoExtractor):
'playlist_mincount': 26,
}]
def _entries(self, app_id, app_name, data_props):
for trailer in traverse_obj(data_props, (
'trailers', lambda _, v: str_or_none(v['id']),
)):
movie_id = str_or_none(trailer['id'])
thumbnails = []
for thumbnail_url in traverse_obj(trailer, (
('poster', 'thumbnail'), {url_or_none},
)):
thumbnails.append({
'url': thumbnail_url,
**parse_resolution(url_basename(thumbnail_url)),
})
formats = []
if hls_manifest := traverse_obj(trailer, ('hlsManifest', {url_or_none})):
formats.extend(self._extract_m3u8_formats(
hls_manifest, app_id, 'mp4', m3u8_id='hls', fatal=False))
for dash_manifest in traverse_obj(trailer, ('dashManifests', ..., {url_or_none})):
formats.extend(self._extract_mpd_formats(
dash_manifest, app_id, mpd_id='dash', fatal=False))
self._remove_duplicate_formats(formats)
yield {
'id': join_nonempty(app_id, movie_id),
'title': join_nonempty(app_name, 'video', movie_id, delim=' '),
'formats': formats,
'series': app_name,
'series_id': app_id,
'thumbnails': thumbnails,
}
def _real_extract(self, url):
app_id = self._match_id(url)
@@ -45,32 +75,13 @@ class SteamIE(InfoExtractor):
self._set_cookie('store.steampowered.com', 'lastagecheckage', '1-January-2000')
webpage = self._download_webpage(url, app_id)
app_name = traverse_obj(webpage, ({find_element(cls='apphub_AppName')}, {clean_html}))
data_props = traverse_obj(webpage, (
{find_element(cls='gamehighlight_desktopcarousel', html=True)},
{extract_attributes}, 'data-props', {json.loads}, {dict}))
app_name = traverse_obj(data_props, ('appName', {clean_html}))
entries = []
for data_prop in traverse_obj(webpage, (
{find_elements(cls='highlight_player_item highlight_movie', html=True)},
..., {extract_attributes}, 'data-props', {json.loads}, {dict},
)):
formats = []
if hls_manifest := traverse_obj(data_prop, ('hlsManifest', {url_or_none})):
formats.extend(self._extract_m3u8_formats(
hls_manifest, app_id, 'mp4', m3u8_id='hls', fatal=False))
for dash_manifest in traverse_obj(data_prop, ('dashManifests', ..., {url_or_none})):
formats.extend(self._extract_mpd_formats(
dash_manifest, app_id, mpd_id='dash', fatal=False))
movie_id = traverse_obj(data_prop, ('id', {trim_str(start='highlight_movie_')}))
entries.append({
'id': movie_id,
'title': join_nonempty(app_name, 'video', movie_id, delim=' '),
'formats': formats,
'series': app_name,
'series_id': app_id,
'thumbnail': traverse_obj(data_prop, ('screenshot', {url_or_none})),
})
return self.playlist_result(entries, app_id, app_name)
return self.playlist_result(
self._entries(app_id, app_name, data_props), app_id, app_name)
class SteamCommunityIE(InfoExtractor):
plugins/youtube_download/yt_dlp/extractor/streaks.py
+5 -1
View File
@@ -22,7 +22,7 @@ class StreaksBaseIE(InfoExtractor):
_GEO_BYPASS = False
_GEO_COUNTRIES = ['JP']
def _extract_from_streaks_api(self, project_id, media_id, headers=None, query=None, ssai=False):
def _extract_from_streaks_api(self, project_id, media_id, headers=None, query=None, ssai=False, live_from_start=False):
try:
response = self._download_json(
self._API_URL_TEMPLATE.format('playback', project_id, media_id, ''),
@@ -83,6 +83,10 @@ class StreaksBaseIE(InfoExtractor):
fmts, subs = self._extract_m3u8_formats_and_subtitles(
src_url, media_id, 'mp4', m3u8_id='hls', fatal=False, live=is_live, query=query)
for fmt in fmts:
if live_from_start:
fmt.setdefault('downloader_options', {}).update({'ffmpeg_args': ['-live_start_index', '0']})
fmt['is_from_start'] = True
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
plugins/youtube_download/yt_dlp/extractor/tarangplus.py
+2 -1
View File
@@ -11,6 +11,7 @@ from ..utils import (
OnDemandPagedList,
clean_html,
extract_attributes,
url_or_none,
urljoin,
)
from ..utils.traversal import (
@@ -128,7 +129,7 @@ class TarangPlusVideoIE(TarangPlusBaseIE):
**traverse_obj(metadata, {
'id': ('content_id', {str}),
'title': ('title', {str}),
'thumbnail': ('image', {str}),
'thumbnail': ('image', {url_or_none}),
}),
**traverse_obj(hidden_inputs_data, {
'id': ('content_id', {str}),
plugins/youtube_download/yt_dlp/extractor/teachable.py
+2 -2
View File
@@ -102,7 +102,7 @@ class TeachableIE(TeachableBaseIE):
_WORKING = False
_VALID_URL = r'''(?x)
(?:
{}https?://(?P<site_t>[^/]+)|
{}https?://(?P<site_t>[a-zA-Z0-9.-]+)|
https?://(?:www\.)?(?P<site>{})
)
/courses/[^/]+/lectures/(?P<id>\d+)
@@ -211,7 +211,7 @@ class TeachableIE(TeachableBaseIE):
class TeachableCourseIE(TeachableBaseIE):
_VALID_URL = r'''(?x)
(?:
{}https?://(?P<site_t>[^/]+)|
{}https?://(?P<site_t>[a-zA-Z0-9.-]+)|
https?://(?:www\.)?(?P<site>{})
)
/(?:courses|p)/(?:enrolled/)?(?P<id>[^/?#&]+)
plugins/youtube_download/yt_dlp/extractor/tele5.py
+35 -28
View File
@@ -9,39 +9,39 @@ class Tele5IE(DiscoveryPlusBaseIE):
_VALID_URL = r'https?://(?:www\.)?tele5\.de/(?P<parent_slug>[\w-]+)/(?P<slug_a>[\w-]+)(?:/(?P<slug_b>[\w-]+))?'
_TESTS = [{
# slug_a and slug_b
'url': 'https://tele5.de/mediathek/stargate-atlantis/quarantane',
'url': 'https://tele5.de/mediathek/star-trek-enterprise/vox-sola',
'info_dict': {
'id': '6852024',
'id': '4140114',
'ext': 'mp4',
'title': 'Quarantäne',
'description': 'md5:6af0373bd0fcc4f13e5d47701903d675',
'episode': 'Episode 73',
'episode_number': 73,
'season': 'Season 4',
'season_number': 4,
'series': 'Stargate Atlantis',
'upload_date': '20240525',
'timestamp': 1716643200,
'duration': 2503.2,
'thumbnail': 'https://eu1-prod-images.disco-api.com/2024/05/21/c81fcb45-8902-309b-badb-4e6d546b575d.jpeg',
'creators': ['Tele5'],
'title': 'Vox Sola',
'description': 'md5:329d115f74324d4364efc1a11c4ea7c9',
'duration': 2542.76,
'thumbnail': r're:https://[^/.]+\.disco-api\.com/.+\.jpe?g',
'tags': [],
'creators': ['Tele5'],
'series': 'Star Trek - Enterprise',
'season': 'Season 1',
'season_number': 1,
'episode': 'Episode 22',
'episode_number': 22,
'timestamp': 1770491100,
'upload_date': '20260207',
},
}, {
# only slug_a
'url': 'https://tele5.de/mediathek/inside-out',
'url': 'https://tele5.de/mediathek/30-miles-from-nowhere-im-wald-hoert-dich-niemand-schreien',
'info_dict': {
'id': '6819502',
'id': '4102641',
'ext': 'mp4',
'title': 'Inside out',
'description': 'md5:7e5f32ed0be5ddbd27713a34b9293bfd',
'series': 'Inside out',
'upload_date': '20240523',
'timestamp': 1716494400,
'duration': 5343.4,
'thumbnail': 'https://eu1-prod-images.disco-api.com/2024/05/15/181eba3c-f9f0-3faf-b14d-0097050a3aa4.jpeg',
'creators': ['Tele5'],
'title': '30 Miles from Nowhere - Im Wald hört dich niemand schreien',
'description': 'md5:0b731539f39ee186ebcd9dd444a86fc2',
'duration': 4849.96,
'thumbnail': r're:https://[^/.]+\.disco-api\.com/.+\.jpe?g',
'tags': [],
'creators': ['Tele5'],
'series': '30 Miles from Nowhere - Im Wald hört dich niemand schreien',
'timestamp': 1770417300,
'upload_date': '20260206',
},
}, {
# playlist
@@ -50,20 +50,27 @@ class Tele5IE(DiscoveryPlusBaseIE):
'id': 'mediathek-schlefaz',
},
'playlist_mincount': 3,
'skip': 'Dead link',
}]
def _real_extract(self, url):
parent_slug, slug_a, slug_b = self._match_valid_url(url).group('parent_slug', 'slug_a', 'slug_b')
playlist_id = join_nonempty(parent_slug, slug_a, slug_b, delim='-')
query = {'environment': 'tele5', 'v': '2'}
query = {
'include': 'default',
'filter[environment]': 'tele5',
'v': '2',
}
if not slug_b:
endpoint = f'page/{slug_a}'
query['parent_slug'] = parent_slug
else:
endpoint = f'videos/{slug_b}'
query['filter[show.slug]'] = slug_a
cms_data = self._download_json(f'https://de-api.loma-cms.com/feloma/{endpoint}/', playlist_id, query=query)
endpoint = f'shows/{slug_a}'
query['filter[video.slug]'] = slug_b
cms_data = self._download_json(f'https://public.aurora.enhanced.live/site/{endpoint}/', playlist_id, query=query)
return self.playlist_result(map(
functools.partial(self._get_disco_api_info, url, disco_host='eu1-prod.disco-api.com', realm='dmaxde', country='DE'),
plugins/youtube_download/yt_dlp/extractor/thechosen.py
+118
View File
@@ -0,0 +1,118 @@
from .common import InfoExtractor
from .frontro import FrontroGroupBaseIE
from ..utils import (
determine_ext,
int_or_none,
url_or_none,
)
from ..utils.traversal import traverse_obj
class TheChosenIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?watch\.thechosen\.tv/(?:video|watch)/(?P<id>[0-9]+)'
_TESTS = [{
'url': 'https://watch.thechosen.tv/video/184683594325',
'md5': '3f878b689588c71b38ec9943c54ff5b0',
'info_dict': {
'id': '184683594325',
'ext': 'mp4',
'title': 'Season 3 Episode 2: Two by Two',
'description': 'md5:174c373756ecc8df46b403f4fcfbaf8c',
'duration': 4212,
'thumbnail': 'https://cas.global.ssl.fastly.net/hls-10-4/184683594325/thumbnail.png',
},
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://watch.thechosen.tv/video/184683596189',
'md5': 'd581562f9d29ce82f5b7770415334151',
'info_dict': {
'id': '184683596189',
'ext': 'mp4',
'title': 'Season 4 Episode 8: Humble',
'description': 'md5:20a57bead43da1cf77cd5b0fe29bbc76',
'duration': 5092,
'thumbnail': 'https://cdn.thechosen.media/videos/cmkvu7nn500nhfm0wpgmm6180/thumbnail.jpg',
},
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://watch.thechosen.tv/video/184683621748',
'info_dict': {
'id': '184683621748',
'ext': 'mp4',
'title': 'Season 5 Episode 2: House of Cards',
'description': 'md5:55b389cbb4b7a01d8c2d837102905617',
'duration': 3086,
'thumbnail': 'https://cdn.thechosen.media/videos/cmkolt4el000afd5zd6x0aeph/thumbnail.jpg',
},
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://watch.thechosen.tv/video/184683621750',
'info_dict': {
'id': '184683621750',
'ext': 'mp4',
'title': 'Season 5 Episode 3: Woes',
'description': 'md5:90ca3cc41316a965fd1cd3d5b3458784',
'duration': 3519,
'thumbnail': 'https://cdn.thechosen.media/videos/cmkoltsl8000dfd5z3luid3mg/thumbnail.jpg',
},
'params': {'skip_download': 'm3u8'},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
metadata = self._download_json(f'https://api.watch.thechosen.tv/v1/videos/{video_id}', video_id)
formats, subtitles = [], {}
for fmt_url in traverse_obj(metadata, ('details', 'video', ..., 'url', {url_or_none})):
ext = determine_ext(fmt_url)
if ext == 'm3u8':
fmts, subs = self._extract_m3u8_formats_and_subtitles(fmt_url, video_id, 'mp4', fatal=False)
elif ext == 'mpd':
fmts, subs = self._extract_mpd_formats_and_subtitles(fmt_url, video_id, fatal=False)
else:
self.report_warning(f'Skipping unsupported format extension "{ext}"', video_id=video_id)
continue
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
thumbnails = []
for thumb_id, thumb_url in traverse_obj(metadata, (
('thumbs', 'thumbnails'), {dict.items}, lambda _, v: url_or_none(v[1]),
)):
thumbnails.append({
'id': thumb_id,
'url': thumb_url,
})
return {
'id': video_id,
**traverse_obj(metadata, ({
'title': ('title', {str}),
'description': ('description', {str}),
'duration': ('duration', {int_or_none}),
})),
'thumbnails': thumbnails,
'formats': formats,
'subtitles': subtitles,
}
class TheChosenGroupIE(FrontroGroupBaseIE):
_WORKING = False
_CHANNEL_ID = '12884901895'
_VIDEO_EXTRACTOR = TheChosenIE
_VIDEO_URL_TMPL = 'https://watch.thechosen.tv/watch/%s'
_VALID_URL = r'https?://(?:www\.)?watch\.thechosen\.tv/group/(?P<id>[0-9]+)'
_TESTS = [{
'url': 'https://watch.thechosen.tv/group/309237658592',
'info_dict': {
'id': '309237658592',
'title': 'Season 3',
'timestamp': 1746203969,
'upload_date': '20250502',
'modified_timestamp': int,
'modified_date': str,
},
'playlist_count': 8,
}]
plugins/youtube_download/yt_dlp/extractor/tiktok.py
+97 -25
View File
@@ -1,4 +1,6 @@
import base64
import functools
import hashlib
import itertools
import json
import random
@@ -15,6 +17,7 @@ from ..utils import (
UnsupportedError,
UserNotLive,
determine_ext,
extract_attributes,
filter_dict,
format_field,
int_or_none,
@@ -25,13 +28,13 @@ from ..utils import (
qualities,
srt_subtitles_timecode,
str_or_none,
traverse_obj,
truncate_string,
try_call,
try_get,
url_or_none,
urlencode_postdata,
)
from ..utils.traversal import find_element, require, traverse_obj
class TikTokBaseIE(InfoExtractor):
@@ -217,38 +220,107 @@ class TikTokBaseIE(InfoExtractor):
raise ExtractorError('Unable to extract aweme detail info', video_id=aweme_id)
return self._parse_aweme_video_app(aweme_detail)
def _solve_challenge_and_set_cookies(self, webpage):
challenge_data = traverse_obj(webpage, (
{find_element(id='cs', html=True)}, {extract_attributes}, 'class',
filter, {lambda x: f'{x}==='}, {base64.b64decode}, {json.loads}))
if not challenge_data:
if 'Please wait...' in webpage:
raise ExtractorError('Unable to extract challenge data')
raise ExtractorError('Unexpected response from webpage request')
self.to_screen('Solving JS challenge using native Python implementation')
expected_digest = traverse_obj(challenge_data, (
'v', 'c', {str}, {base64.b64decode},
{require('challenge expected digest')}))
base_hash = traverse_obj(challenge_data, (
'v', 'a', {str}, {base64.b64decode},
{hashlib.sha256}, {require('challenge base hash')}))
for i in range(1_000_001):
number = str(i).encode()
test_hash = base_hash.copy()
test_hash.update(number)
if test_hash.digest() == expected_digest:
challenge_data['d'] = base64.b64encode(number).decode()
break
else:
raise ExtractorError('Unable to solve JS challenge')
wci_cookie_value = base64.b64encode(
json.dumps(challenge_data, separators=(',', ':')).encode()).decode()
# At time of writing, the wci cookie name was `_wafchallengeid`
wci_cookie_name = traverse_obj(webpage, (
{find_element(id='wci', html=True)}, {extract_attributes},
'class', {require('challenge cookie name')}))
# At time of writing, the **optional** rci cookie name was `waforiginalreid`
rci_cookie_name = traverse_obj(webpage, (
{find_element(id='rci', html=True)}, {extract_attributes}, 'class'))
rci_cookie_value = traverse_obj(webpage, (
{find_element(id='rs', html=True)}, {extract_attributes}, 'class'))
# Actual JS sets Max-Age=1 for the cookies, but we'll manually clear them later instead
expire_time = int(time.time()) + (self.get_param('sleep_interval_requests') or 0) + 120
self._set_cookie('.tiktok.com', wci_cookie_name, wci_cookie_value, expire_time=expire_time)
if rci_cookie_name and rci_cookie_value:
self._set_cookie('.tiktok.com', rci_cookie_name, rci_cookie_value, expire_time=expire_time)
return wci_cookie_name, rci_cookie_name
def _extract_web_data_and_status(self, url, video_id, fatal=True):
video_data, status = {}, -1
res = self._download_webpage_handle(url, video_id, fatal=fatal, impersonate=True)
if res is False:
def get_webpage(note='Downloading webpage'):
res = self._download_webpage_handle(url, video_id, note, fatal=fatal, impersonate=True)
if res is False:
return False
webpage, urlh = res
if urllib.parse.urlparse(urlh.url).path == '/login':
message = 'TikTok is requiring login for access to this content'
if fatal:
self.raise_login_required(message)
self.report_warning(f'{message}. {self._login_hint()}', video_id=video_id)
return False
return webpage
webpage = get_webpage()
if webpage is False:
return video_data, status
webpage, urlh = res
if urllib.parse.urlparse(urlh.url).path == '/login':
message = 'TikTok is requiring login for access to this content'
universal_data = self._get_universal_data(webpage, video_id)
if not universal_data:
try:
cookie_names = self._solve_challenge_and_set_cookies(webpage)
except ExtractorError as e:
if fatal:
raise
self.report_warning(e.orig_msg, video_id=video_id)
return video_data, status
webpage = get_webpage(note='Downloading webpage with challenge cookie')
# Manually clear challenge cookies that should expire immediately after webpage request
for cookie_name in filter(None, cookie_names):
self.cookiejar.clear(domain='.tiktok.com', path='/', name=cookie_name)
if webpage is False:
return video_data, status
universal_data = self._get_universal_data(webpage, video_id)
if not universal_data:
message = 'Unable to extract universal data for rehydration'
if fatal:
self.raise_login_required(message)
self.report_warning(f'{message}. {self._login_hint()}')
raise ExtractorError(message)
self.report_warning(message, video_id=video_id)
return video_data, status
if universal_data := self._get_universal_data(webpage, video_id):
self.write_debug('Found universal data for rehydration')
status = traverse_obj(universal_data, ('webapp.video-detail', 'statusCode', {int})) or 0
video_data = traverse_obj(universal_data, ('webapp.video-detail', 'itemInfo', 'itemStruct', {dict}))
elif sigi_data := self._get_sigi_state(webpage, video_id):
self.write_debug('Found sigi state data')
status = traverse_obj(sigi_data, ('VideoPage', 'statusCode', {int})) or 0
video_data = traverse_obj(sigi_data, ('ItemModule', video_id, {dict}))
elif next_data := self._search_nextjs_data(webpage, video_id, default={}):
self.write_debug('Found next.js data')
status = traverse_obj(next_data, ('props', 'pageProps', 'statusCode', {int})) or 0
video_data = traverse_obj(next_data, ('props', 'pageProps', 'itemInfo', 'itemStruct', {dict}))
elif fatal:
raise ExtractorError('Unable to extract webpage video data')
status = traverse_obj(universal_data, ('webapp.video-detail', 'statusCode', {int})) or 0
video_data = traverse_obj(universal_data, ('webapp.video-detail', 'itemInfo', 'itemStruct', {dict}))
if not traverse_obj(video_data, ('video', {dict})) and traverse_obj(video_data, ('isContentClassified', {bool})):
message = 'This post may not be comfortable for some audiences. Log in for access'
plugins/youtube_download/yt_dlp/extractor/truth.py
+2 -1
View File
@@ -51,7 +51,8 @@ class TruthIE(InfoExtractor):
def _real_extract(self, url):
video_id = self._match_id(url)
status = self._download_json(f'https://truthsocial.com/api/v1/statuses/{video_id}', video_id)
status = self._download_json(
f'https://truthsocial.com/api/v1/statuses/{video_id}', video_id, impersonate=True)
uploader_id = strip_or_none(traverse_obj(status, ('account', 'username')))
return {
'id': video_id,
plugins/youtube_download/yt_dlp/extractor/tver.py
+113
View File
@@ -4,6 +4,7 @@ from .streaks import StreaksBaseIE
from ..utils import (
ExtractorError,
GeoRestrictedError,
clean_html,
int_or_none,
join_nonempty,
make_archive_id,
@@ -11,7 +12,9 @@ from ..utils import (
str_or_none,
strip_or_none,
time_seconds,
unified_timestamp,
update_url_query,
url_or_none,
)
from ..utils.traversal import require, traverse_obj
@@ -257,3 +260,113 @@ class TVerIE(StreaksBaseIE):
'id': video_id,
'_old_archive_ids': [make_archive_id('BrightcoveNew', brightcove_id)] if brightcove_id else None,
}
class TVerOlympicIE(StreaksBaseIE):
IE_NAME = 'tver:olympic'
_API_BASE = 'https://olympic-data.tver.jp/api'
_VALID_URL = r'https?://(?:www\.)?tver\.jp/olympic/milanocortina2026/(?P<type>live|video)/play/(?P<id>\w+)'
_TESTS = [{
'url': 'https://tver.jp/olympic/milanocortina2026/video/play/3b1d4462150b42558d9cc8aabb5238d0/',
'info_dict': {
'id': '3b1d4462150b42558d9cc8aabb5238d0',
'ext': 'mp4',
'title': '【開会式】ぎゅっと凝縮ハイライト',
'display_id': 'ref:3b1d4462150b42558d9cc8aabb5238d0',
'duration': 712.045,
'live_status': 'not_live',
'modified_date': r're:\d{8}',
'modified_timestamp': int,
'tags': 'count:1',
'thumbnail': r're:https://.+\.(?:jpg|png)',
'timestamp': 1770420187,
'upload_date': '20260206',
'uploader_id': 'tver-olympic',
},
}, {
'url': 'https://tver.jp/olympic/milanocortina2026/live/play/glts313itwvj/',
'info_dict': {
'id': 'glts313itwvj',
'ext': 'mp4',
'title': '開会式ハイライト',
'channel_id': 'ntv',
'display_id': 'ref:sp_260207_spc_01_dvr',
'duration': 7680,
'live_status': 'was_live',
'modified_date': r're:\d{8}',
'modified_timestamp': int,
'thumbnail': r're:https://.+\.(?:jpg|png)',
'timestamp': 1770420300,
'upload_date': '20260206',
'uploader_id': 'tver-olympic-live',
},
}]
def _real_extract(self, url):
video_type, video_id = self._match_valid_url(url).group('type', 'id')
live_from_start = self.get_param('live_from_start')
if video_type == 'live':
project_id = 'tver-olympic-live'
api_key = 'a35ebb1ca7d443758dc7fcc5d99b1f72'
olympic_data = traverse_obj(self._download_json(
f'{self._API_BASE}/live/{video_id}', video_id), ('contents', 'live', {dict}))
media_id = traverse_obj(olympic_data, ('video_id', {str}))
now = time_seconds()
start_timestamp_str = traverse_obj(olympic_data, ('onair_start_date', {str}))
start_timestamp = unified_timestamp(start_timestamp_str, tz_offset=9)
if not start_timestamp:
raise ExtractorError('Unable to extract on-air start time')
end_timestamp = traverse_obj(olympic_data, (
'onair_end_date', {unified_timestamp(tz_offset=9)}, {require('on-air end time')}))
if now < start_timestamp:
self.raise_no_formats(
f'This program is scheduled to start at {start_timestamp_str} JST', expected=True)
return {
'id': video_id,
'live_status': 'is_upcoming',
'release_timestamp': start_timestamp,
}
elif start_timestamp <= now < end_timestamp:
live_status = 'is_live'
if live_from_start:
media_id += '_dvr'
elif end_timestamp <= now:
dvr_end_timestamp = traverse_obj(olympic_data, (
'dvr_end_date', {unified_timestamp(tz_offset=9)}))
if dvr_end_timestamp and now < dvr_end_timestamp:
live_status = 'was_live'
media_id += '_dvr'
else:
raise ExtractorError(
'This program is no longer available', expected=True)
else:
project_id = 'tver-olympic'
api_key = '4b55a4db3cce4ad38df6dd8543e3e46a'
media_id = video_id
live_status = 'not_live'
olympic_data = traverse_obj(self._download_json(
f'{self._API_BASE}/video/{video_id}', video_id), ('contents', 'video', {dict}))
return {
**self._extract_from_streaks_api(project_id, f'ref:{media_id}', {
'Origin': 'https://tver.jp',
'Referer': 'https://tver.jp/',
'X-Streaks-Api-Key': api_key,
}, live_from_start=live_from_start),
**traverse_obj(olympic_data, {
'title': ('title', {clean_html}, filter),
'alt_title': ('sub_title', {clean_html}, filter),
'channel': ('channel', {clean_html}, filter),
'channel_id': ('channel_id', {clean_html}, filter),
'description': (('description', 'description_l', 'description_s'), {clean_html}, filter, any),
'timestamp': ('onair_start_date', {unified_timestamp(tz_offset=9)}),
'thumbnail': (('picture_l_url', 'picture_m_url', 'picture_s_url'), {url_or_none}, any),
}),
'id': video_id,
'live_status': live_status,
}
plugins/youtube_download/yt_dlp/extractor/tvo.py
+152
View File
@@ -0,0 +1,152 @@
import json
import urllib.parse
from .brightcove import BrightcoveNewIE
from .common import InfoExtractor
from ..utils import (
clean_html,
int_or_none,
parse_duration,
parse_iso8601,
smuggle_url,
str_or_none,
url_or_none,
)
from ..utils.traversal import (
require,
traverse_obj,
trim_str,
)
class TvoIE(InfoExtractor):
IE_NAME = 'TVO'
_VALID_URL = r'https?://(?:www\.)?tvo\.org/video(?:/documentaries)?/(?P<id>[\w-]+)'
_TESTS = [{
'url': 'https://www.tvo.org/video/how-can-ontario-survive-the-trade-war',
'info_dict': {
'id': '6377531034112',
'ext': 'mp4',
'title': 'How Can Ontario Survive the Trade War?',
'description': 'md5:e7455d9cd4b6b1270141922044161457',
'display_id': 'how-can-ontario-survive-the-trade-war',
'duration': 3531,
'episode': 'How Can Ontario Survive the Trade War?',
'episode_id': 'how-can-ontario-survive-the-trade-war',
'episode_number': 1,
'season': 'Season 1',
'season_number': 1,
'series': 'TVO at AMO',
'series_id': 'tvo-at-amo',
'tags': 'count:17',
'thumbnail': r're:https?://.+',
'timestamp': 1756944016,
'upload_date': '20250904',
'uploader_id': '18140038001',
},
}, {
'url': 'https://www.tvo.org/video/documentaries/the-pitch',
'info_dict': {
'id': '6382500333112',
'ext': 'mp4',
'title': 'The Pitch',
'categories': ['Documentaries'],
'description': 'md5:9d4246b70dce772a3a396c4bd84c8506',
'display_id': 'the-pitch',
'duration': 5923,
'episode': 'The Pitch',
'episode_id': 'the-pitch',
'episode_number': 1,
'season': 'Season 1',
'season_number': 1,
'series': 'The Pitch',
'series_id': 'the-pitch',
'tags': 'count:8',
'thumbnail': r're:https?://.+',
'timestamp': 1762693216,
'upload_date': '20251109',
'uploader_id': '18140038001',
},
}, {
'url': 'https://www.tvo.org/video/documentaries/valentines-day',
'info_dict': {
'id': '6387298331112',
'ext': 'mp4',
'title': 'Valentine\'s Day',
'categories': ['Documentaries'],
'description': 'md5:b142149beb2d3a855244816c50cd2f14',
'display_id': 'valentines-day',
'duration': 3121,
'episode': 'Valentine\'s Day',
'episode_id': 'valentines-day',
'episode_number': 2,
'season': 'Season 1',
'season_number': 1,
'series': 'How We Celebrate',
'series_id': 'how-we-celebrate',
'tags': 'count:6',
'thumbnail': r're:https?://.+',
'timestamp': 1770386416,
'upload_date': '20260206',
'uploader_id': '18140038001',
},
}]
BRIGHTCOVE_URL_TEMPLATE = 'https://players.brightcove.net/18140038001/default_default/index.html?videoId=%s'
def _real_extract(self, url):
display_id = self._match_id(url)
video_data = self._download_json(
'https://hmy0rc1bo2.execute-api.ca-central-1.amazonaws.com/graphql',
display_id, headers={'Content-Type': 'application/json'},
data=json.dumps({
'operationName': 'getVideo',
'variables': {'slug': urllib.parse.urlparse(url).path.rstrip('/')},
'query': '''query getVideo($slug: String) {
getTVOOrgVideo(slug: $slug) {
contentCategory
description
length
program {
nodeUrl
title
}
programOrder
publishedAt
season
tags
thumbnail
title
videoSource {
brightcoveRefId
}
}
}''',
}, separators=(',', ':')).encode(),
)['data']['getTVOOrgVideo']
brightcove_id = traverse_obj(video_data, (
'videoSource', 'brightcoveRefId', {str_or_none}, {require('Brightcove ID')}))
return {
'_type': 'url_transparent',
'ie_key': BrightcoveNewIE.ie_key(),
'url': smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, {'geo_countries': ['CA']}),
'display_id': display_id,
'episode_id': display_id,
**traverse_obj(video_data, {
'title': ('title', {clean_html}, filter),
'categories': ('contentCategory', {clean_html}, filter, all, filter),
'description': ('description', {clean_html}, filter),
'duration': ('length', {parse_duration}),
'episode': ('title', {clean_html}, filter),
'episode_number': ('programOrder', {int_or_none}),
'season_number': ('season', {int_or_none}),
'tags': ('tags', ..., {clean_html}, filter),
'thumbnail': ('thumbnail', {url_or_none}),
'timestamp': ('publishedAt', {parse_iso8601}),
}),
**traverse_obj(video_data, ('program', {
'series': ('title', {clean_html}, filter),
'series_id': ('nodeUrl', {clean_html}, {trim_str(start='/programs/')}, filter),
})),
}
plugins/youtube_download/yt_dlp/extractor/twitch.py
+3 -3
View File
@@ -43,10 +43,10 @@ class TwitchBaseIE(InfoExtractor):
_OPERATION_HASHES = {
'CollectionSideBar': '016e1e4ccee0eb4698eb3bf1a04dc1c077fb746c78c82bac9a8f0289658fbd1a',
'FilterableVideoTower_Videos': '67004f7881e65c297936f32c75246470629557a393788fb5a69d6d9a25a8fd5f',
'ClipsCards__User': '90c33f5e6465122fba8f9371e2a97076f9ed06c6fed3788d002ab9eba8f91d88',
'ShareClipRenderStatus': '1844261bb449fa51e6167040311da4a7a5f1c34fe71c71a3e0c4f551bc30c698',
'ClipsCards__User': '1cd671bfa12cec480499c087319f26d21925e9695d1f80225aae6a4354f23088',
'ShareClipRenderStatus': '0a02bb974443b576f5579aab0fef1d4b7f44e58a8a256f0c5adfead0db70640f',
'ChannelCollectionsContent': '5247910a19b1cd2b760939bf4cba4dcbd3d13bdf8c266decd16956f6ef814077',
'StreamMetadata': 'b57f9b910f8cd1a4659d894fe7550ccc81ec9052c01e438b290fd66a040b9b93',
'StreamMetadata': 'ad022ca32220d5523d03a23cbcb5beaa1e0999889c1f8f78f9f2520dafb5cae6',
'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
'VideoPreviewOverlay': '9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f',
'VideoMetadata': '45111672eea2e507f8ba44d101a61862f9c56b11dee09a15634cb75cb9b9084d',
plugins/youtube_download/yt_dlp/extractor/twitter.py
+10 -6
View File
@@ -131,11 +131,15 @@ class TwitterBaseIE(InfoExtractor):
video_id, headers=headers, query=query, expected_status=allowed_status,
note=f'Downloading {"GraphQL" if graphql else "legacy API"} JSON')
if result.get('errors'):
errors = ', '.join(set(traverse_obj(result, ('errors', ..., 'message', {str}))))
if errors and 'not authorized' in errors:
self.raise_login_required(remove_end(errors, '.'))
raise ExtractorError(f'Error(s) while querying API: {errors or "Unknown error"}')
if error_msg := ', '.join(set(traverse_obj(result, ('errors', ..., 'message', {str})))):
# Errors with the message 'Dependency: Unspecified' are a false positive
# See https://github.com/yt-dlp/yt-dlp/issues/15963
if error_msg.lower() == 'dependency: unspecified':
self.write_debug(f'Ignoring Twitter API error: "{error_msg}"')
elif 'not authorized' in error_msg.lower():
self.raise_login_required(remove_end(error_msg, '.'))
else:
raise ExtractorError(f'Error(s) while querying API: {error_msg or "Unknown error"}')
return result
@@ -1078,7 +1082,7 @@ class TwitterIE(TwitterBaseIE):
raise ExtractorError(f'Twitter API says: {cause or "Unknown error"}', expected=True)
elif typename == 'TweetUnavailable':
reason = result.get('reason')
if reason == 'NsfwLoggedOut':
if reason in ('NsfwLoggedOut', 'NsfwViewerHasNoStatedAge'):
self.raise_login_required('NSFW tweet requires authentication')
elif reason == 'Protected':
self.raise_login_required('You are not authorized to view this protected tweet')
plugins/youtube_download/yt_dlp/extractor/unsupported.py
+26
View File
@@ -66,6 +66,11 @@ class KnownDRMIE(UnsupportedInfoExtractor):
r'rtlmost\.hu',
r'plus\.rtl\.de(?!/podcast/)',
r'mediasetinfinity\.es',
r'tv5mondeplus\.com',
r'tv\.rakuten\.co\.jp',
r'watch\.telusoriginals\.com',
r'video\.unext\.jp',
r'www\.web\.nhk',
)
_TESTS = [{
@@ -226,6 +231,27 @@ class KnownDRMIE(UnsupportedInfoExtractor):
}, {
'url': 'https://www.mediasetinfinity.es/',
'only_matching': True,
}, {
# https://github.com/yt-dlp/yt-dlp/issues/14743
'url': 'https://www.tv5mondeplus.com/',
'only_matching': True,
}, {
# https://github.com/yt-dlp/yt-dlp/issues/8821
'url': 'https://tv.rakuten.co.jp/content/519554/',
'only_matching': True,
}, {
# https://github.com/yt-dlp/yt-dlp/issues/9851
'url': 'https://watch.telusoriginals.com/play?assetID=fruit-is-ripe',
'only_matching': True,
}, {
# https://github.com/yt-dlp/yt-dlp/issues/13220
# https://github.com/yt-dlp/yt-dlp/issues/14564
'url': 'https://video.unext.jp/play/SID0062010/ED00337407',
'only_matching': True,
}, {
# https://github.com/yt-dlp/yt-dlp/issues/14620
'url': 'https://www.web.nhk/tv/an/72hours/pl/series-tep-W3W8WRN8M3/ep/QW8ZY6146V',
'only_matching': True,
}]
def _real_extract(self, url):
plugins/youtube_download/yt_dlp/extractor/vimeo.py
+16 -1
View File
@@ -49,7 +49,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
'Cannot download embed-only video without embedding URL. Please call yt-dlp '
'with the URL of the page that embeds this video.')
_DEFAULT_CLIENT = 'web'
_DEFAULT_CLIENT = 'macos'
_DEFAULT_AUTHED_CLIENT = 'web'
_CLIENT_HEADERS = {
'Accept': 'application/vnd.vimeo.*+json; version=3.4.10',
@@ -87,6 +87,21 @@ class VimeoBaseInfoExtractor(InfoExtractor):
'upload', 'transcode', 'is_playable', 'has_audio',
),
},
'macos': {
'CACHE_KEY': 'oauth-token-macos',
'CACHE_ONLY': False,
'VIEWER_JWT': False,
'REQUIRES_AUTH': False,
'AUTH': 'NDc1N2JlN2Y5ZjZmMjU3NzE3NTRkZTg1NmY2YzU2MTI0OTFlNjJiYjpwVUNDWUlBZmZqSHhQcndBYWxGMzgyYys2NkN5d1JrREJZZXdPcEdsU05tdjFlVVo2aE1lYk9GcWE3ZW9KVldlYnFlOWh5Vno5UWtpUGJ5empYZFBpYkFwV0FFTnB5VWV4ZEh3aHZnRUNEL0VySnBzTmFraDdNbS9nMXhWanhIcw==',
'USER_AGENT': 'Vimeo/1.6.3 (com.vimeo.mac; build:251121.142637.0; macOS 13.7.8) Alamofire/5.9.0 VimeoNetworking/5.0.0',
'VIDEOS_FIELDS': (
'uri', 'name', 'description', 'type', 'link', 'player_embed_url', 'duration', 'width',
'language', 'height', 'embed', 'created_time', 'modified_time', 'release_time', 'content_rating',
'content_rating_class', 'rating_mod_locked', 'license', 'privacy', 'pictures', 'tags', 'stats',
'categories', 'uploader', 'metadata', 'user', 'files', 'download', 'app', 'play', 'status',
'resource_key', 'badge', 'upload', 'transcode', 'is_playable', 'has_audio',
),
},
'web': {
'CACHE_ONLY': False,
'VIEWER_JWT': True,
plugins/youtube_download/yt_dlp/extractor/visir.py
+116
View File
@@ -0,0 +1,116 @@
import re
from .common import InfoExtractor
from ..utils import (
UnsupportedError,
clean_html,
int_or_none,
js_to_json,
month_by_name,
url_or_none,
urljoin,
)
from ..utils.traversal import find_element, traverse_obj
class VisirIE(InfoExtractor):
IE_DESC = 'Vísir'
_VALID_URL = r'https?://(?:www\.)?visir\.is/(?P<type>k|player)/(?P<id>[\da-f-]+)(?:/(?P<slug>[\w.-]+))?'
_EMBED_REGEX = [rf'<iframe[^>]+src=["\'](?P<url>{_VALID_URL})']
_TESTS = [{
'url': 'https://www.visir.is/k/eabb8f7f-ad87-46fb-9469-a0f1dc0fc4bc-1769022963988',
'info_dict': {
'id': 'eabb8f7f-ad87-46fb-9469-a0f1dc0fc4bc-1769022963988',
'ext': 'mp4',
'title': 'Sveppi og Siggi Þór mestu skaphundarnir',
'categories': ['island-i-dag'],
'description': 'md5:e06bd6a0cd8bdde328ad8cf00d3d4df6',
'duration': 792,
'thumbnail': r're:https?://www\.visir\.is/.+',
'upload_date': '20260121',
'view_count': int,
},
}, {
'url': 'https://www.visir.is/k/b0a88e02-eceb-4270-855c-8328b76b9d81-1763979306704/tonlistarborgin-reykjavik',
'info_dict': {
'id': 'b0a88e02-eceb-4270-855c-8328b76b9d81-1763979306704',
'ext': 'mp4',
'title': 'Tónlistarborgin Reykjavík',
'categories': ['tonlist'],
'description': 'md5:47237589dc95dbde55dfbb163396f88a',
'display_id': 'tonlistarborgin-reykjavik',
'duration': 81,
'thumbnail': r're:https?://www\.visir\.is/.+',
'upload_date': '20251124',
'view_count': int,
},
}, {
'url': 'https://www.visir.is/player/0cd5709e-6870-46d0-aaaf-0ae637de94f1-1770060083580',
'info_dict': {
'id': '0cd5709e-6870-46d0-aaaf-0ae637de94f1-1770060083580',
'ext': 'mp4',
'title': 'Sportpakkinn 2. febrúar 2026',
'categories': ['sportpakkinn'],
'display_id': 'sportpakkinn-2.-februar-2026',
'duration': 293,
'thumbnail': r're:https?://www\.visir\.is/.+',
'upload_date': '20260202',
'view_count': int,
},
}]
_WEBPAGE_TESTS = [{
'url': 'https://www.visir.is/g/20262837896d/segir-von-brigdin-med-prinsessuna-rista-djupt',
'info_dict': {
'id': '9ad5e58a-f26f-49f7-8b1d-68f0629485b7-1770059257365',
'ext': 'mp4',
'title': 'Norðmenn tala ekki um annað en prinsessuna',
'categories': ['frettir'],
'description': 'md5:53e2623ae79e1355778c14f5b557a0cd',
'display_id': 'nordmenn-tala-ekki-um-annad-en-prinsessuna',
'duration': 138,
'thumbnail': r're:https?://www\.visir\.is/.+',
'upload_date': '20260202',
'view_count': int,
},
}]
def _real_extract(self, url):
video_type, video_id, display_id = self._match_valid_url(url).group('type', 'id', 'slug')
webpage = self._download_webpage(url, video_id)
if video_type == 'player':
real_url = self._og_search_url(webpage)
if not self.suitable(real_url) or self._match_valid_url(real_url).group('type') == 'player':
raise UnsupportedError(real_url)
return self.url_result(real_url, self.ie_key())
upload_date = None
date_elements = traverse_obj(webpage, (
{find_element(cls='article-item__date')}, {clean_html}, filter, {str.split}))
if date_elements and len(date_elements) == 3:
day, month, year = date_elements
day = int_or_none(day.rstrip('.'))
month = month_by_name(month, 'is')
if day and month and re.fullmatch(r'[0-9]{4}', year):
upload_date = f'{year}{month:02d}{day:02d}'
player = self._search_json(
r'App\.Player\.Init\(', webpage, video_id, 'player', transform_source=js_to_json)
m3u8_url = traverse_obj(player, ('File', {urljoin('https://vod.visir.is/')}))
return {
'id': video_id,
'display_id': display_id,
'formats': self._extract_m3u8_formats(m3u8_url, video_id, 'mp4'),
'upload_date': upload_date,
**traverse_obj(webpage, ({find_element(cls='article-item press-ads')}, {
'description': ({find_element(cls='-large')}, {clean_html}, filter),
'view_count': ({find_element(cls='article-item__viewcount')}, {clean_html}, {int_or_none}),
})),
**traverse_obj(player, {
'title': ('Title', {clean_html}),
'categories': ('Categoryname', {clean_html}, filter, all, filter),
'duration': ('MediaDuration', {int_or_none}),
'thumbnail': ('Image', {url_or_none}),
}),
}
plugins/youtube_download/yt_dlp/extractor/vk.py
+31 -7
View File
@@ -1,6 +1,7 @@
import collections
import hashlib
import re
import urllib.parse
from .common import InfoExtractor
from .dailymotion import DailymotionIE
@@ -8,6 +9,7 @@ from .odnoklassniki import OdnoklassnikiIE
from .sibnet import SibnetEmbedIE
from .vimeo import VimeoIE
from .youtube import YoutubeIE
from ..jsinterp import JSInterpreter
from ..utils import (
ExtractorError,
UserNotLive,
@@ -36,16 +38,38 @@ class VKBaseIE(InfoExtractor):
def _download_webpage_handle(self, url_or_request, video_id, *args, fatal=True, **kwargs):
response = super()._download_webpage_handle(url_or_request, video_id, *args, fatal=fatal, **kwargs)
challenge_url, cookie = response[1].url if response else '', None
if challenge_url.startswith('https://vk.com/429.html?'):
cookie = self._get_cookies(challenge_url).get('hash429')
if not cookie:
if response is False:
return response
hash429 = hashlib.md5(cookie.value.encode('ascii')).hexdigest()
webpage, urlh = response
challenge_url = urlh.url
if urllib.parse.urlparse(challenge_url).path != '/challenge.html':
return response
self.to_screen(join_nonempty(
video_id and f'[{video_id}]',
'Received a JS challenge response',
delim=' '))
challenge_hash = traverse_obj(challenge_url, (
{parse_qs}, 'hash429', -1, {require('challenge hash')}))
func_code = self._search_regex(
r'(?s)var\s+salt\s*=\s*\(\s*function\s*\(\)\s*(\{.+?\})\s*\)\(\);\s*var\s+hash',
webpage, 'JS challenge salt function')
jsi = JSInterpreter(f'function salt() {func_code}')
salt = jsi.extract_function('salt')([])
self.write_debug(f'Generated salt with native JS interpreter: {salt}')
key_hash = hashlib.md5(f'{challenge_hash}:{salt}'.encode()).hexdigest()
self.write_debug(f'JS challenge key hash: {key_hash}')
# Request with the challenge key and the response should set a 'solution429' cookie
self._request_webpage(
update_url_query(challenge_url, {'key': hash429}), video_id, fatal=fatal,
note='Resolving WAF challenge', errnote='Failed to bypass WAF challenge')
update_url_query(challenge_url, {'key': key_hash}), video_id,
'Submitting JS challenge solution', 'Unable to solve JS challenge', fatal=True)
return super()._download_webpage_handle(url_or_request, video_id, *args, fatal=True, **kwargs)
def _perform_login(self, username, password):
plugins/youtube_download/yt_dlp/extractor/volejtv.py
+149 -22
View File
@@ -1,40 +1,167 @@
import functools
from .common import InfoExtractor
from ..utils import (
InAdvancePagedList,
int_or_none,
join_nonempty,
orderedSet,
str_or_none,
strftime_or_none,
unified_timestamp,
url_or_none,
)
from ..utils.traversal import (
require,
traverse_obj,
)
class VolejTVIE(InfoExtractor):
_VALID_URL = r'https?://volej\.tv/video/(?P<id>\d+)'
class VolejTVBaseIE(InfoExtractor):
TBR_HEIGHT_MAPPING = {
'6000': 1080,
'2400': 720,
'1500': 480,
'800': 360,
}
def _call_api(self, endpoint, display_id, query=None):
return self._download_json(
f'https://api-volejtv-prod.apps.okd4.devopsie.cloud/api/{endpoint}',
display_id, query=query)
class VolejTVIE(VolejTVBaseIE):
IE_NAME = 'volejtv:match'
_VALID_URL = r'https?://volej\.tv/match/(?P<id>\d+)'
_TESTS = [{
'url': 'https://volej.tv/video/725742/',
'url': 'https://volej.tv/match/270579',
'info_dict': {
'id': '725742',
'id': '270579',
'ext': 'mp4',
'description': 'Zápas VK Královo Pole vs VK Prostějov 10.12.2022 v 19:00 na Volej.TV',
'thumbnail': 'https://volej.tv/images/og/16/17186/og.png',
'title': 'VK Královo Pole vs VK Prostějov',
'title': 'SWE-CZE (2024-06-16)',
'categories': ['ženy'],
'series': 'ZLATÁ EVROPSKÁ VOLEJBALOVÁ LIGA',
'season': '2023-2024',
'timestamp': 1718553600,
'upload_date': '20240616',
},
}, {
'url': 'https://volej.tv/video/725605/',
'url': 'https://volej.tv/match/487520',
'info_dict': {
'id': '725605',
'id': '487520',
'ext': 'mp4',
'thumbnail': 'https://volej.tv/images/og/15/17185/og.png',
'title': 'VK Lvi Praha vs VK Euro Sitex Příbram',
'description': 'Zápas VK Lvi Praha vs VK Euro Sitex Příbram 11.12.2022 v 19:00 na Volej.TV',
'thumbnail': r're:https://.+\.(png|jpeg)',
'title': 'FRA-CZE (2024-09-06)',
'categories': ['mládež'],
'series': 'Mistrovství Evropy do 20 let',
'season': '2024-2025',
'timestamp': 1725627600,
'upload_date': '20240906',
},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
json_data = self._search_json(
r'<\s*!\[CDATA[^=]+=', webpage, 'CDATA', video_id)
formats, subtitle = self._extract_m3u8_formats_and_subtitles(
json_data['urls']['hls'], video_id)
return {
json_data = self._call_api(f'match/{video_id}', video_id)
formats = []
for video in traverse_obj(json_data, ('videos', 0, 'qualities', lambda _, v: url_or_none(v['cloud_front_path']))):
formats.append(traverse_obj(video, {
'url': 'cloud_front_path',
'tbr': ('quality', {int_or_none}),
'format_id': ('id', {str_or_none}),
'height': ('quality', {self.TBR_HEIGHT_MAPPING.get}),
}))
data = {
'id': video_id,
'title': self._html_search_meta(['og:title', 'twitter:title'], webpage),
'thumbnail': self._html_search_meta(['og:image', 'twitter:image'], webpage),
'description': self._html_search_meta(['description', 'og:description', 'twitter:description'], webpage),
**traverse_obj(json_data, {
'series': ('competition_name', {str}),
'season': ('season', {str}),
'timestamp': ('match_time', {unified_timestamp}),
'categories': ('category', ('title'), {str}, filter, all, filter),
'thumbnail': ('poster', {url_or_none}),
}),
'formats': formats,
'subtitles': subtitle,
}
teams = orderedSet(traverse_obj(json_data, ('teams', ..., 'shortcut', {str})))
if len(teams) > 2 and 'FIN' in teams:
teams.remove('FIN')
data['title'] = join_nonempty(
join_nonempty(*teams, delim='-'),
strftime_or_none(data.get('timestamp'), '(%Y-%m-%d)'),
delim=' ')
return data
class VolejTVPlaylistBaseIE(VolejTVBaseIE):
"""Subclasses must set _API_FILTER, _PAGE_SIZE"""
def _get_page(self, playlist_id, page):
return self._call_api(
f'match/{self._API_FILTER}/{playlist_id}', playlist_id,
query={'page': page + 1, 'take': self._PAGE_SIZE, 'order': 'DESC'})
def _entries(self, playlist_id, first_page_data, page):
entries = first_page_data if page == 0 else self._get_page(playlist_id, page)
for match_id in traverse_obj(entries, ('data', ..., 'id')):
yield self.url_result(f'https://volej.tv/match/{match_id}', VolejTVIE)
class VolejTVClubPlaylistIE(VolejTVPlaylistBaseIE):
IE_NAME = 'volejtv:club'
_VALID_URL = r'https?://volej\.tv/klub/(?P<id>\d+)'
_TESTS = [{
'url': 'https://volej.tv/klub/1173',
'info_dict': {
'id': '1173',
'title': 'VK Jihostroj České Budějovice',
},
'playlist_mincount': 30,
}]
_API_FILTER = 'by-team-id-paginated'
_PAGE_SIZE = 6
def _real_extract(self, url):
playlist_id = self._match_id(url)
title = self._call_api(f'team/show/{playlist_id}', playlist_id)['title']
first_page_data = self._get_page(playlist_id, 0)
total_pages = traverse_obj(first_page_data, ('meta', 'pageCount', {int}, {require('page count')}))
return self.playlist_result(InAdvancePagedList(
functools.partial(self._entries, playlist_id, first_page_data),
total_pages, self._PAGE_SIZE), playlist_id, title)
class VolejTVCategoryPlaylistIE(VolejTVPlaylistBaseIE):
IE_NAME = 'volejtv:category'
_VALID_URL = r'https?://volej\.tv/kategorie/(?P<id>[^/$?]+)'
_TESTS = [{
'url': 'https://volej.tv/kategorie/chance-cesky-pohar',
'info_dict': {
'id': 'chance-cesky-pohar',
'title': 'Chance Český pohár',
},
'playlist_mincount': 30,
}]
_API_FILTER = 'by-category-id-paginated'
_PAGE_SIZE = 10
def _get_category(self, playlist_id):
categories = self._call_api('category', playlist_id)
for category in traverse_obj(categories, (lambda _, v: v['slug'] and v['id'] and v['title'])):
if category['slug'] == playlist_id:
return category['id'], category['title']
def _real_extract(self, url):
playlist_id = self._match_id(url)
category_id, title = self._get_category(playlist_id)
first_page_data = self._get_page(category_id, 0)
total_pages = traverse_obj(first_page_data, ('meta', 'pageCount', {int}, {require('page count')}))
return self.playlist_result(InAdvancePagedList(
functools.partial(self._entries, category_id, first_page_data),
total_pages, self._PAGE_SIZE), playlist_id, title)
plugins/youtube_download/yt_dlp/extractor/wat.py
+1 -1
View File
@@ -76,7 +76,7 @@ class WatIE(InfoExtractor):
if error_code == 'GEOBLOCKED':
self.raise_geo_restricted(error_desc, video_info.get('geoList'))
elif error_code == 'DELIVERY_ERROR':
if traverse_obj(video_data, ('delivery', 'code')) == 500:
if traverse_obj(video_data, ('delivery', 'code')) in (403, 500):
self.report_drm(video_id)
error_desc = join_nonempty(
error_desc, traverse_obj(video_data, ('delivery', 'error', {str})), delim=': ')
plugins/youtube_download/yt_dlp/extractor/whyp.py
+23 -13
View File
@@ -1,6 +1,8 @@
from .common import InfoExtractor
from ..utils import (
float_or_none,
int_or_none,
parse_iso8601,
str_or_none,
traverse_obj,
url_or_none,
@@ -11,17 +13,19 @@ class WhypIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?whyp\.it/tracks/(?P<id>\d+)'
_TESTS = [{
'url': 'https://www.whyp.it/tracks/18337/home-page-example-track-b4kq7',
'md5': 'c1187b42ebf8605284e3dc92aeb33d16',
'md5': '02fd96427acd9547445979bf0496b013',
'info_dict': {
'url': 'https://cdn.whyp.it/50eb17cc-e9ff-4e18-b89b-dc9206a95cb1.mp3',
'id': '18337',
'title': 'Home Page Example Track',
'description': 'md5:bd758000fb93f3159339c852b5b9133c',
'ext': 'mp3',
'duration': 52.82,
'title': 'Example Track',
'display_id': 'example-track',
'description': 'md5:e0b1bcf1d267dc1a0f15efff09c8f297',
'ext': 'flac',
'duration': 135.63,
'timestamp': 1643216583,
'upload_date': '20220126',
'uploader': 'Brad',
'uploader_id': '1',
'thumbnail': 'https://cdn.whyp.it/a537bb36-3373-4c61-96c8-27fc1b2f427a.jpg',
'thumbnail': 'https://cdn.whyp.it/6ad0bbd9-577d-42bb-9b61-2a4f57f647eb.jpg',
},
}, {
'url': 'https://www.whyp.it/tracks/18337',
@@ -34,17 +38,23 @@ class WhypIE(InfoExtractor):
data = self._search_nuxt_data(webpage, unique_id)['rawTrack']
return {
'url': data['audio_url'],
'id': unique_id,
'formats': [{
'url': data[f'{prefix}_url'],
'format_id': prefix,
'filesize': int_or_none(data.get(f'{prefix}_size')),
'vcodec': 'none',
'quality': 10 if prefix == 'lossless' else -1,
'http_headers': {'Referer': 'https://whyp.it/'},
} for prefix in ('audio', 'lossy', 'lossless') if url_or_none(data.get(f'{prefix}_url'))],
**traverse_obj(data, {
'title': 'title',
'title': ('title', {str}),
'display_id': ('slug', {str}),
'description': 'description',
'duration': ('duration', {float_or_none}),
'uploader': ('user', 'username'),
'timestamp': ('created_at', {parse_iso8601}),
'uploader': ('user', 'username', {str}),
'uploader_id': ('user', 'id', {str_or_none}),
'thumbnail': ('artwork_url', {url_or_none}),
}),
'ext': 'mp3',
'vcodec': 'none',
'http_headers': {'Referer': 'https://whyp.it/'},
}
plugins/youtube_download/yt_dlp/extractor/xhamster.py
+59 -43
View File
@@ -3,6 +3,7 @@ import re
import urllib.parse
from .common import InfoExtractor
from ..jsinterp import int_to_int32
from ..utils import (
ExtractorError,
clean_html,
@@ -20,73 +21,69 @@ from ..utils import (
)
def to_signed_32(n):
return n % ((-1 if n < 0 else 1) * 2**32)
class _ByteGenerator:
def __init__(self, algo_id, seed):
try:
self._algorithm = getattr(self, f'_algo{algo_id}')
except AttributeError:
raise ExtractorError(f'Unknown algorithm ID "{algo_id}"')
self._s = to_signed_32(seed)
self._s = int_to_int32(seed)
def _algo1(self, s):
# LCG (a=1664525, c=1013904223, m=2^32)
# Ref: https://en.wikipedia.org/wiki/Linear_congruential_generator
s = self._s = to_signed_32(s * 1664525 + 1013904223)
s = self._s = int_to_int32(s * 1664525 + 1013904223)
return s
def _algo2(self, s):
# xorshift32
# Ref: https://en.wikipedia.org/wiki/Xorshift
s = to_signed_32(s ^ (s << 13))
s = to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 17))
s = self._s = to_signed_32(s ^ (s << 5))
s = int_to_int32(s ^ (s << 13))
s = int_to_int32(s ^ ((s & 0xFFFFFFFF) >> 17))
s = self._s = int_to_int32(s ^ (s << 5))
return s
def _algo3(self, s):
# Weyl Sequence (k≈2^32*φ, m=2^32) + MurmurHash3 (fmix32)
# Ref: https://en.wikipedia.org/wiki/Weyl_sequence
# https://commons.apache.org/proper/commons-codec/jacoco/org.apache.commons.codec.digest/MurmurHash3.java.html
s = self._s = to_signed_32(s + 0x9e3779b9)
s = to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 16))
s = to_signed_32(s * to_signed_32(0x85ebca77))
s = to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 13))
s = to_signed_32(s * to_signed_32(0xc2b2ae3d))
return to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 16))
s = self._s = int_to_int32(s + 0x9e3779b9)
s = int_to_int32(s ^ ((s & 0xFFFFFFFF) >> 16))
s = int_to_int32(s * int_to_int32(0x85ebca77))
s = int_to_int32(s ^ ((s & 0xFFFFFFFF) >> 13))
s = int_to_int32(s * int_to_int32(0xc2b2ae3d))
return int_to_int32(s ^ ((s & 0xFFFFFFFF) >> 16))
def _algo4(self, s):
# Custom scrambling function involving a left rotation (ROL)
s = self._s = to_signed_32(s + 0x6d2b79f5)
s = to_signed_32((s << 7) | ((s & 0xFFFFFFFF) >> 25)) # ROL 7
s = to_signed_32(s + 0x9e3779b9)
s = to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 11))
return to_signed_32(s * 0x27d4eb2d)
s = self._s = int_to_int32(s + 0x6d2b79f5)
s = int_to_int32((s << 7) | ((s & 0xFFFFFFFF) >> 25)) # ROL 7
s = int_to_int32(s + 0x9e3779b9)
s = int_to_int32(s ^ ((s & 0xFFFFFFFF) >> 11))
return int_to_int32(s * 0x27d4eb2d)
def _algo5(self, s):
# xorshift variant with a final addition
s = to_signed_32(s ^ (s << 7))
s = to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 9))
s = to_signed_32(s ^ (s << 8))
s = self._s = to_signed_32(s + 0xa5a5a5a5)
s = int_to_int32(s ^ (s << 7))
s = int_to_int32(s ^ ((s & 0xFFFFFFFF) >> 9))
s = int_to_int32(s ^ (s << 8))
s = self._s = int_to_int32(s + 0xa5a5a5a5)
return s
def _algo6(self, s):
# LCG (a=0x2c9277b5, c=0xac564b05) with a variable right shift scrambler
s = self._s = to_signed_32(s * to_signed_32(0x2c9277b5) + to_signed_32(0xac564b05))
s2 = to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 18))
s = self._s = int_to_int32(s * int_to_int32(0x2c9277b5) + int_to_int32(0xac564b05))
s2 = int_to_int32(s ^ ((s & 0xFFFFFFFF) >> 18))
shift = (s & 0xFFFFFFFF) >> 27 & 31
return to_signed_32((s2 & 0xFFFFFFFF) >> shift)
return int_to_int32((s2 & 0xFFFFFFFF) >> shift)
def _algo7(self, s):
# Weyl Sequence (k=0x9e3779b9) + custom multiply-xor-shift mixing function
s = self._s = to_signed_32(s + to_signed_32(0x9e3779b9))
e = to_signed_32(s ^ (s << 5))
e = to_signed_32(e * to_signed_32(0x7feb352d))
e = to_signed_32(e ^ ((e & 0xFFFFFFFF) >> 15))
return to_signed_32(e * to_signed_32(0x846ca68b))
s = self._s = int_to_int32(s + int_to_int32(0x9e3779b9))
e = int_to_int32(s ^ (s << 5))
e = int_to_int32(e * int_to_int32(0x7feb352d))
e = int_to_int32(e ^ ((e & 0xFFFFFFFF) >> 15))
return int_to_int32(e * int_to_int32(0x846ca68b))
def __next__(self):
return self._algorithm(self._s) & 0xFF
@@ -213,16 +210,9 @@ class XHamsterIE(InfoExtractor):
'only_matching': True,
}]
def _decipher_format_url(self, format_url, format_id):
parsed_url = urllib.parse.urlparse(format_url)
hex_string, path_remainder = self._search_regex(
r'^/(?P<hex>[0-9a-fA-F]{12,})(?P<rem>[/,].+)$', parsed_url.path, 'url components',
default=(None, None), group=('hex', 'rem'))
if not hex_string:
self.report_warning(f'Skipping format "{format_id}": unsupported URL format')
return None
_VALID_HEX_RE = r'[0-9a-fA-F]{12,}'
def _decipher_hex_string(self, hex_string, format_id):
byte_data = bytes.fromhex(hex_string)
seed = int.from_bytes(byte_data[1:5], byteorder='little', signed=True)
@@ -232,7 +222,33 @@ class XHamsterIE(InfoExtractor):
self.report_warning(f'Skipping format "{format_id}": {e.msg}')
return None
deciphered = bytearray(byte ^ next(byte_gen) for byte in byte_data[5:]).decode('latin-1')
return bytearray(byte ^ next(byte_gen) for byte in byte_data[5:]).decode('latin-1')
def _decipher_format_url(self, format_url, format_id):
# format_url can be hex ciphertext or a URL with a hex ciphertext segment
if re.fullmatch(self._VALID_HEX_RE, format_url):
return self._decipher_hex_string(format_url, format_id)
elif not url_or_none(format_url):
if re.fullmatch(r'[0-9a-fA-F]+', format_url):
# Hex strings that are too short are expected, so we don't want to warn
self.write_debug(f'Skipping dummy ciphertext for "{format_id}": {format_url}')
else:
# Something has likely changed on the site's end, so we need to warn
self.report_warning(f'Skipping format "{format_id}": invalid ciphertext')
return None
parsed_url = urllib.parse.urlparse(format_url)
hex_string, path_remainder = self._search_regex(
rf'^/(?P<hex>{self._VALID_HEX_RE})(?P<rem>[/,].+)$', parsed_url.path, 'url components',
default=(None, None), group=('hex', 'rem'))
if not hex_string:
self.report_warning(f'Skipping format "{format_id}": unsupported URL format')
return None
deciphered = self._decipher_hex_string(hex_string, format_id)
if not deciphered:
return None
return parsed_url._replace(path=f'/{deciphered}{path_remainder}').geturl()
@@ -252,7 +268,7 @@ class XHamsterIE(InfoExtractor):
display_id = mobj.group('display_id') or mobj.group('display_id_2')
desktop_url = re.sub(r'^(https?://(?:.+?\.)?)m\.', r'\1', url)
webpage, urlh = self._download_webpage_handle(desktop_url, video_id)
webpage, urlh = self._download_webpage_handle(desktop_url, video_id, impersonate=True)
error = self._html_search_regex(
r'<div[^>]+id=["\']videoClosed["\'][^>]*>(.+?)</div>',
plugins/youtube_download/yt_dlp/extractor/youtube/__init__.py
+1 -2
View File
@@ -16,7 +16,7 @@ from ._redirect import (
YoutubeYtBeIE,
YoutubeYtUserIE,
)
from ._search import YoutubeMusicSearchURLIE, YoutubeSearchDateIE, YoutubeSearchIE, YoutubeSearchURLIE
from ._search import YoutubeMusicSearchURLIE, YoutubeSearchIE, YoutubeSearchURLIE
from ._tab import YoutubePlaylistIE, YoutubeTabBaseInfoExtractor, YoutubeTabIE
from ._video import YoutubeIE
@@ -39,7 +39,6 @@ for _cls in [
YoutubeYtBeIE,
YoutubeYtUserIE,
YoutubeMusicSearchURLIE,
YoutubeSearchDateIE,
YoutubeSearchIE,
YoutubeSearchURLIE,
YoutubePlaylistIE,
plugins/youtube_download/yt_dlp/extractor/youtube/_base.py
+29 -57
View File
@@ -99,12 +99,11 @@ INNERTUBE_CLIENTS = {
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'WEB',
'clientVersion': '2.20250925.01.00',
'clientVersion': '2.20260114.08.00',
},
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 1,
'SUPPORTS_COOKIES': True,
'SUPPORTS_AD_PLAYBACK_CONTEXT': True,
**WEB_PO_TOKEN_POLICIES,
},
# Safari UA returns pre-merged video+audio 144p/240p/360p/720p/1080p HLS formats
@@ -112,20 +111,19 @@ INNERTUBE_CLIENTS = {
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'WEB',
'clientVersion': '2.20250925.01.00',
'clientVersion': '2.20260114.08.00',
'userAgent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15,gzip(gfe)',
},
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 1,
'SUPPORTS_COOKIES': True,
'SUPPORTS_AD_PLAYBACK_CONTEXT': True,
**WEB_PO_TOKEN_POLICIES,
},
'web_embedded': {
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'WEB_EMBEDDED_PLAYER',
'clientVersion': '1.20250923.21.00',
'clientVersion': '1.20260115.01.00',
},
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 56,
@@ -136,7 +134,7 @@ INNERTUBE_CLIENTS = {
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'WEB_REMIX',
'clientVersion': '1.20250922.03.00',
'clientVersion': '1.20260114.03.00',
},
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
@@ -166,7 +164,7 @@ INNERTUBE_CLIENTS = {
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'WEB_CREATOR',
'clientVersion': '1.20250922.03.00',
'clientVersion': '1.20260114.05.00',
},
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
@@ -195,9 +193,9 @@ INNERTUBE_CLIENTS = {
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'ANDROID',
'clientVersion': '20.10.38',
'clientVersion': '21.02.35',
'androidSdkVersion': 30,
'userAgent': 'com.google.android.youtube/20.10.38 (Linux; U; Android 11) gzip',
'userAgent': 'com.google.android.youtube/21.02.35 (Linux; U; Android 11) gzip',
'osName': 'Android',
'osVersion': '11',
},
@@ -223,21 +221,8 @@ INNERTUBE_CLIENTS = {
},
'PLAYER_PO_TOKEN_POLICY': PlayerPoTokenPolicy(required=False, recommended=True),
},
# Doesn't require a PoToken for some reason
'android_sdkless': {
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'ANDROID',
'clientVersion': '20.10.38',
'userAgent': 'com.google.android.youtube/20.10.38 (Linux; U; Android 11) gzip',
'osName': 'Android',
'osVersion': '11',
},
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
'REQUIRE_JS_PLAYER': False,
},
# YouTube Kids videos aren't returned on this client for some reason
# "Made for kids" videos aren't available with this client
# Using a clientVersion>1.65 may return SABR streams only
'android_vr': {
'INNERTUBE_CONTEXT': {
'client': {
@@ -260,10 +245,10 @@ INNERTUBE_CLIENTS = {
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'IOS',
'clientVersion': '20.10.4',
'clientVersion': '21.02.3',
'deviceMake': 'Apple',
'deviceModel': 'iPhone16,2',
'userAgent': 'com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)',
'userAgent': 'com.google.ios.youtube/21.02.3 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)',
'osName': 'iPhone',
'osVersion': '18.3.2.22D82',
},
@@ -291,7 +276,7 @@ INNERTUBE_CLIENTS = {
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'MWEB',
'clientVersion': '2.20250925.01.00',
'clientVersion': '2.20260115.01.00',
# mweb previously did not require PO Token with this UA
'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)',
},
@@ -322,24 +307,24 @@ INNERTUBE_CLIENTS = {
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'TVHTML5',
'clientVersion': '7.20250923.13.00',
'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version',
'clientVersion': '7.20260114.12.00',
# See: https://github.com/youtube/cobalt/blob/main/cobalt/browser/user_agent/user_agent_platform_info.cc#L506
'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/25.lts.30.1034943-gold (unlike Gecko), Unknown_TV_Unknown_0/Unknown (Unknown, Unknown)',
},
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 7,
'SUPPORTS_COOKIES': True,
# See: https://github.com/youtube/cobalt/blob/main/cobalt/browser/user_agent/user_agent_platform_info.cc#L506
'AUTHENTICATED_USER_AGENT': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/25.lts.30.1034943-gold (unlike Gecko), Unknown_TV_Unknown_0/Unknown (Unknown, Unknown)',
},
'tv_downgraded': {
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'TVHTML5',
'clientVersion': '5.20251105',
'clientVersion': '5.20260114',
'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version',
},
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 7,
'REQUIRE_AUTH': True,
'SUPPORTS_COOKIES': True,
},
'tv_simply': {
@@ -365,20 +350,6 @@ INNERTUBE_CLIENTS = {
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 75,
},
# This client now requires sign-in for every video
# It was previously an age-gate workaround for videos that were `playable_in_embed`
# It may still be useful if signed into an EU account that is not age-verified
'tv_embedded': {
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
'clientVersion': '2.0',
},
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 85,
'REQUIRE_AUTH': True,
'SUPPORTS_COOKIES': True,
},
}
@@ -397,7 +368,7 @@ def short_client_name(client_name):
def _fix_embedded_ytcfg(ytcfg):
ytcfg['INNERTUBE_CONTEXT'].setdefault('thirdParty', {}).update({
'embedUrl': 'https://www.youtube.com/', # Can be any valid URL
'embedUrl': 'https://www.reddit.com/', # Can be any valid non-YouTube URL
})
@@ -418,7 +389,6 @@ def build_innertube_clients():
ytcfg.setdefault('SUPPORTS_COOKIES', False)
ytcfg.setdefault('SUPPORTS_AD_PLAYBACK_CONTEXT', False)
ytcfg.setdefault('PLAYER_PARAMS', None)
ytcfg.setdefault('AUTHENTICATED_USER_AGENT', None)
ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
_, base_client, variant = _split_innertube_client(client)
@@ -703,14 +673,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
_YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
def _get_default_ytcfg(self, client='web'):
ytcfg = copy.deepcopy(INNERTUBE_CLIENTS[client])
# Currently, only the tv client needs to use an alternative user-agent when logged-in
if ytcfg.get('AUTHENTICATED_USER_AGENT') and self.is_authenticated:
client_context = ytcfg.setdefault('INNERTUBE_CONTEXT', {}).setdefault('client', {})
client_context['userAgent'] = ytcfg['AUTHENTICATED_USER_AGENT']
return ytcfg
return copy.deepcopy(INNERTUBE_CLIENTS[client])
def _get_innertube_host(self, client='web'):
return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
@@ -994,16 +957,25 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
url = {
'mweb': 'https://m.youtube.com',
'web': 'https://www.youtube.com',
'web_safari': 'https://www.youtube.com',
'web_music': 'https://music.youtube.com',
'web_creator': 'https://studio.youtube.com',
'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1',
'tv': 'https://www.youtube.com/tv',
}.get(client)
if not url:
return {}
default_ytcfg = self._get_default_ytcfg(client)
if default_ytcfg['REQUIRE_AUTH'] and not self.is_authenticated:
return {}
webpage = self._download_webpage_with_retries(
url, video_id, note=f'Downloading {client.replace("_", " ").strip()} client config',
headers=traverse_obj(self._get_default_ytcfg(client), {
headers=traverse_obj(default_ytcfg, {
'User-Agent': ('INNERTUBE_CONTEXT', 'client', 'userAgent', {str}),
'Referer': ('INNERTUBE_CONTEXT', 'thirdParty', 'embedUrl', {str}),
}))
ytcfg = self.extract_ytcfg(video_id, webpage) or {}
plugins/youtube_download/yt_dlp/extractor/youtube/_search.py
-15
View File
@@ -28,21 +28,6 @@ class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
}]
class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
_SEARCH_KEY = 'ytsearchdate'
IE_DESC = 'YouTube search, newest videos first'
_SEARCH_PARAMS = 'CAISAhAB8AEB' # Videos only, sorted by date
_TESTS = [{
'url': 'ytsearchdate5:youtube-dl test video',
'playlist_count': 5,
'info_dict': {
'id': 'youtube-dl test video',
'title': 'youtube-dl test video',
},
}]
class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
IE_DESC = 'YouTube search URLs with sorting and filter support'
IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
plugins/youtube_download/yt_dlp/extractor/youtube/_tab.py
+2 -2
View File
@@ -81,7 +81,7 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
'reelPlayerHeaderSupportedRenderers', 'reelPlayerHeaderRenderer'))
title = self._get_text(renderer, 'title', 'headline') or self._get_text(reel_header_renderer, 'reelTitleText')
description = self._get_text(renderer, 'descriptionSnippet')
description = self._get_text(renderer, 'descriptionSnippet', ('detailedMetadataSnippets', ..., 'snippetText'))
duration = int_or_none(renderer.get('lengthSeconds'))
if duration is None:
@@ -2148,7 +2148,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
f'https://music.youtube.com/playlist?list={item_id[2:]}', YoutubeTabIE, item_id[2:])
elif item_id[:2] == 'MP': # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
mdata = self._extract_tab_endpoint(
f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
f'https://music.youtube.com/browse/{item_id}', item_id, default_client='web_music')
murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
get_all=False, expected_type=str)
if not murl:
plugins/youtube_download/yt_dlp/extractor/youtube/_video.py
+289 -205
View File
@@ -10,7 +10,6 @@ import re
import sys
import threading
import time
import traceback
import urllib.parse
from ._base import (
@@ -63,6 +62,7 @@ from ...utils import (
unescapeHTML,
unified_strdate,
unsmuggle_url,
update_url,
update_url_query,
url_or_none,
urljoin,
@@ -139,17 +139,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
]
_RETURN_TYPE = 'video' # XXX: How to handle multifeed?
_PLAYER_INFO_RE = (
r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/(?:tv-)?player',
r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
)
_SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'srt', 'vtt')
_DEFAULT_CLIENTS = ('tv', 'android_sdkless', 'web')
_DEFAULT_JSLESS_CLIENTS = ('android_sdkless', 'web_safari', 'web')
_DEFAULT_AUTHED_CLIENTS = ('tv_downgraded', 'web_safari', 'web')
_DEFAULT_CLIENTS = ('android_vr', 'web_safari')
_DEFAULT_JSLESS_CLIENTS = ('android_vr',)
_DEFAULT_AUTHED_CLIENTS = ('tv_downgraded', 'web_safari')
# Premium does not require POT (except for subtitles)
_DEFAULT_PREMIUM_CLIENTS = ('tv_downgraded', 'web_creator', 'web')
_DEFAULT_PREMIUM_CLIENTS = ('tv_downgraded', 'web_creator')
_WEBPAGE_CLIENTS = ('web', 'web_safari')
_DEFAULT_WEBPAGE_CLIENT = 'web_safari'
_GEO_BYPASS = False
@@ -1443,7 +1440,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'view_count': int,
},
'params': {
'extractor_args': {'youtube': {'player_client': ['tv_embedded']}},
'format': '251-drc',
'skip_download': True,
},
@@ -1690,7 +1686,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'playable_in_embed': True,
'availability': 'public',
'live_status': 'not_live',
'comment_count': 15, # XXX: minimum
'comment_count': 15, # XXX: minimum, but investigate if this changes
'comments': 'count:15',
},
'params': {
'skip_download': True,
@@ -1723,7 +1720,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'playable_in_embed': True,
'availability': 'unlisted',
'live_status': 'not_live',
'comment_count': 9, # XXX: minimum
'comment_count': 9, # XXX: minimum, but investigate if this changes
'comments': 'count:9',
},
'params': {
'skip_download': True,
'getcomments': True,
},
}]
_WEBPAGE_TESTS = [{
@@ -1881,13 +1883,37 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'tce': 'player_ias_tce.vflset/en_US/base.js',
'es5': 'player_es5.vflset/en_US/base.js',
'es6': 'player_es6.vflset/en_US/base.js',
'es6_tcc': 'player_es6_tcc.vflset/en_US/base.js',
'es6_tce': 'player_es6_tce.vflset/en_US/base.js',
'tv': 'tv-player-ias.vflset/tv-player-ias.js',
'tv_es6': 'tv-player-es6.vflset/tv-player-es6.js',
'phone': 'player-plasma-ias-phone-en_US.vflset/base.js',
'tablet': 'player-plasma-ias-tablet-en_US.vflset/base.js',
'house': 'house_brand_player.vflset/en_US/base.js', # Used by Google Drive
}
_INVERSE_PLAYER_JS_VARIANT_MAP = {v: k for k, v in _PLAYER_JS_VARIANT_MAP.items()}
@functools.cached_property
def _player_js_version(self):
return self._configuration_arg('player_js_version', [None])[0] or self._DEFAULT_PLAYER_JS_VERSION
@functools.cached_property
def _webpage_client(self):
webpage_client = self._configuration_arg('webpage_client', [self._DEFAULT_WEBPAGE_CLIENT])[0]
if webpage_client not in self._WEBPAGE_CLIENTS:
self.report_warning(
f'Invalid webpage_client "{webpage_client}" requested; '
f'falling back to {self._DEFAULT_WEBPAGE_CLIENT}', only_once=True)
webpage_client = self._DEFAULT_WEBPAGE_CLIENT
return webpage_client
@functools.cached_property
def _skipped_webpage_data(self):
skipped = set(self._configuration_arg('webpage_skip'))
# If forcing a player version, the webpage player response must be skipped
if self._player_js_version != 'actual':
skipped.add('player_response')
return skipped
@classmethod
def suitable(cls, url):
from yt_dlp.utils import parse_qs
@@ -1913,13 +1939,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
start_time = time.time()
formats = [f for f in formats if f.get('is_from_start')]
def refetch_manifest(format_id, delay):
def refetch_manifest(itag, client_name, delay):
nonlocal formats, start_time, is_live
if time.time() <= start_time + delay:
return
_, _, _, _, prs, player_url = self._initial_extract(
url, smuggled_data, webpage_url, 'web', video_id)
url, smuggled_data, webpage_url, self._webpage_client, video_id)
video_details = traverse_obj(prs, (..., 'videoDetails'), expected_type=dict)
microformats = traverse_obj(
prs, (..., 'microformat', 'playerMicroformatRenderer'),
@@ -1928,20 +1954,20 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
is_live = live_status == 'is_live'
start_time = time.time()
def mpd_feed(format_id, delay):
def mpd_feed(itag, client_name, delay):
"""
@returns (manifest_url, manifest_stream_number, is_live) or None
"""
for retry in self.RetryManager(fatal=False):
with lock:
refetch_manifest(format_id, delay)
refetch_manifest(itag, client_name, delay)
f = next((f for f in formats if f['format_id'] == format_id), None)
f = next((f for f in formats if f.get('_itag') == itag and f.get('_client') == client_name), None)
if not f:
if not is_live:
retry.error = f'{video_id}: Video is no longer live'
else:
retry.error = f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}'
retry.error = f'Cannot find refreshed manifest for format {itag}{bug_reports_message()}'
continue
# Formats from ended premieres will be missing a manifest_url
@@ -1954,7 +1980,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
for f in formats:
f['is_live'] = is_live
gen = functools.partial(self._live_dash_fragments, video_id, f['format_id'],
gen = functools.partial(self._live_dash_fragments, video_id, f['_itag'], f['_client'],
live_start_time, mpd_feed, not is_live and f.copy())
if is_live:
f['fragments'] = gen
@@ -1963,7 +1989,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
f['fragments'] = LazyList(gen({}))
del f['is_from_start']
def _live_dash_fragments(self, video_id, format_id, live_start_time, mpd_feed, manifestless_orig_fmt, ctx):
def _live_dash_fragments(self, video_id, itag, client_name, live_start_time, mpd_feed, manifestless_orig_fmt, ctx):
FETCH_SPAN, MAX_DURATION = 5, 432000
mpd_url, stream_number, is_live = None, None, True
@@ -1987,7 +2013,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
old_mpd_url = mpd_url
last_error = ctx.pop('last_error', None)
expire_fast = immediate or (last_error and isinstance(last_error, HTTPError) and last_error.status == 403)
mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
mpd_url, stream_number, is_live = (mpd_feed(itag, client_name, 5 if expire_fast else 18000)
or (mpd_url, stream_number, False))
if not refresh_sequence:
if expire_fast and not is_live:
@@ -2013,7 +2039,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
_last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
return True, _last_seq
self.write_debug(f'[{video_id}] Generating fragments for format {format_id}')
self.write_debug(f'[{video_id}] Generating fragments for format {itag}')
while is_live:
fetch_time = time.time()
if no_fragment_score > 30:
@@ -2075,15 +2101,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
def _get_player_js_version(self):
player_js_version = self._configuration_arg('player_js_version', [''])[0] or self._DEFAULT_PLAYER_JS_VERSION
if player_js_version == 'actual':
if self._player_js_version == 'actual':
return None, None
if not re.fullmatch(r'[0-9]{5,}@[0-9a-f]{8,}', player_js_version):
if not re.fullmatch(r'[0-9]{5,}@[0-9a-f]{8,}', self._player_js_version):
self.report_warning(
f'Invalid player JS version "{player_js_version}" specified. '
f'Invalid player JS version "{self._player_js_version}" specified. '
f'It should be "actual" or in the format of STS@HASH', only_once=True)
return None, None
return player_js_version.split('@')
return self._player_js_version.split('@')
def _construct_player_url(self, *, player_id=None, player_url=None):
assert player_id or player_url, '_construct_player_url must take one of player_id or player_url'
@@ -2174,13 +2199,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
@classmethod
def _extract_player_info(cls, player_url):
for player_re in cls._PLAYER_INFO_RE:
id_m = re.search(player_re, player_url)
if id_m:
break
else:
raise ExtractorError(f'Cannot identify player {player_url!r}')
return id_m.group('id')
if m := re.search(r'/s/player/(?P<id>[a-fA-F0-9]{8,})/', player_url):
return m.group('id')
raise ExtractorError(f'Cannot identify player {player_url!r}')
def _load_player(self, video_id, player_url, fatal=True):
player_js_key = self._player_js_cache_key(player_url)
@@ -2193,64 +2214,32 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
self._code_cache[player_js_key] = code
return self._code_cache.get(player_js_key)
def _sig_spec_cache_id(self, player_url, spec_id):
return join_nonempty(self._player_js_cache_key(player_url), str(spec_id))
def _load_player_data_from_cache(self, name, player_url, *cache_keys, use_disk_cache=False):
cache_id = (f'youtube-{name}', self._player_js_cache_key(player_url), *map(str_or_none, cache_keys))
if cache_id in self._player_cache:
return self._player_cache[cache_id]
def _load_sig_spec_from_cache(self, spec_cache_id):
# This is almost identical to _load_player_data_from_cache
# I hate it
if spec_cache_id in self._player_cache:
return self._player_cache[spec_cache_id]
spec = self.cache.load('youtube-sigfuncs', spec_cache_id, min_ver='2025.07.21')
if spec:
self._player_cache[spec_cache_id] = spec
return spec
if not use_disk_cache:
return None
def _store_sig_spec_to_cache(self, spec_cache_id, spec):
if spec_cache_id not in self._player_cache:
self._player_cache[spec_cache_id] = spec
self.cache.store('youtube-sigfuncs', spec_cache_id, spec)
def _load_player_data_from_cache(self, name, player_url):
cache_id = (f'youtube-{name}', self._player_js_cache_key(player_url))
if data := self._player_cache.get(cache_id):
return data
data = self.cache.load(*cache_id, min_ver='2025.07.21')
data = self.cache.load(cache_id[0], join_nonempty(*cache_id[1:]), min_ver='2025.07.21')
if data:
self._player_cache[cache_id] = data
return data
def _cached(self, func, *cache_id):
def inner(*args, **kwargs):
if cache_id not in self._player_cache:
try:
self._player_cache[cache_id] = func(*args, **kwargs)
except ExtractorError as e:
self._player_cache[cache_id] = e
except Exception as e:
self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
ret = self._player_cache[cache_id]
if isinstance(ret, Exception):
raise ret
return ret
return inner
def _store_player_data_to_cache(self, name, player_url, data):
cache_id = (f'youtube-{name}', self._player_js_cache_key(player_url))
def _store_player_data_to_cache(self, data, name, player_url, *cache_keys, use_disk_cache=False):
cache_id = (f'youtube-{name}', self._player_js_cache_key(player_url), *map(str_or_none, cache_keys))
if cache_id not in self._player_cache:
self.cache.store(*cache_id, data)
self._player_cache[cache_id] = data
if use_disk_cache:
self.cache.store(cache_id[0], join_nonempty(*cache_id[1:]), data)
def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
"""
Extract signatureTimestamp (sts)
Required to tell API what sig/player version is in use.
"""
CACHE_ENABLED = False # TODO: enable when preprocessed player JS cache is solved/enabled
player_sts_override = self._get_player_js_version()[0]
if player_sts_override:
@@ -2267,15 +2256,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
self.report_warning(error_msg)
return None
if CACHE_ENABLED and (sts := self._load_player_data_from_cache('sts', player_url)):
# TODO: Pass `use_disk_cache=True` when preprocessed player JS cache is solved
if sts := self._load_player_data_from_cache('sts', player_url):
return sts
if code := self._load_player(video_id, player_url, fatal=fatal):
sts = int_or_none(self._search_regex(
r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
'JS player signature timestamp', group='sts', fatal=fatal))
if CACHE_ENABLED and sts:
self._store_player_data_to_cache('sts', player_url, sts)
if sts:
# TODO: Pass `use_disk_cache=True` when preprocessed player JS cache is solved
self._store_player_data_to_cache(sts, 'sts', player_url)
return sts
@@ -2707,12 +2698,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
return {'contentCheckOk': True, 'racyCheckOk': True}
@classmethod
def _generate_player_context(cls, sts=None, use_ad_playback_context=False):
def _generate_player_context(cls, sts=None, use_ad_playback_context=False, encrypted_context=None):
context = {
'html5Preference': 'HTML5_PREF_WANTS',
}
if sts is not None:
context['signatureTimestamp'] = sts
if encrypted_context:
context['encryptedHostFlags'] = encrypted_context
playback_context = {
'contentPlaybackContext': context,
@@ -2793,7 +2786,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'serializedExperimentFlags', {urllib.parse.parse_qs}))
if 'true' in traverse_obj(experiments, (..., 'html5_generate_content_po_token', -1)):
self.write_debug(
f'{video_id}: Detected experiment to bind GVS PO Token to video id.', only_once=True)
f'{video_id}: Detected experiment to bind GVS PO Token '
f'to video ID for {client} client', only_once=True)
gvs_bind_to_video_id = True
# GVS WebPO Token is bound to visitor_data / Visitor ID when logged out.
@@ -2956,7 +2950,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
self._configuration_arg('use_ad_playback_context', ['false'])[0] != 'false'
and traverse_obj(INNERTUBE_CLIENTS, (client, 'SUPPORTS_AD_PLAYBACK_CONTEXT', {bool})))
yt_query.update(self._generate_player_context(sts, use_ad_playback_context))
# web_embedded player requests may need to include encryptedHostFlags in its contentPlaybackContext.
# This can be detected with the embeds_enable_encrypted_host_flags_enforcement experiemnt flag,
# but there is no harm in including encryptedHostFlags with all web_embedded player requests.
encrypted_context = None
if _split_innertube_client(client)[2] == 'embedded':
encrypted_context = traverse_obj(player_ytcfg, (
'WEB_PLAYER_CONTEXT_CONFIGS', 'WEB_PLAYER_CONTEXT_CONFIG_ID_EMBEDDED_PLAYER', 'encryptedHostFlags'))
yt_query.update(
self._generate_player_context(
sts=sts,
use_ad_playback_context=use_ad_playback_context,
encrypted_context=encrypted_context))
return self._extract_response(
item_id=video_id, ep='player', query=yt_query,
@@ -3075,7 +3081,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
tried_iframe_fallback = True
pr = None
if client == webpage_client and 'player_response' not in self._configuration_arg('webpage_skip'):
if client == webpage_client and 'player_response' not in self._skipped_webpage_data:
pr = initial_pr
visitor_data = visitor_data or self._extract_visitor_data(webpage_ytcfg, initial_pr, player_ytcfg)
@@ -3141,6 +3147,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
else:
prs.append(pr)
if (
# Is this a "made for kids" video that can't be downloaded with android_vr?
client == 'android_vr' and self._is_unplayable(pr)
and webpage and 'made for kids' in webpage
# ...and is a JS runtime is available?
and any(p.is_available() for p in self._jsc_director.providers.values())
):
append_client('web_embedded')
# web_embedded can work around age-gate and age-verification for some embeddable videos
if self._is_agegated(pr) and variant != 'web_embedded':
append_client(f'web_embedded.{base_client}')
@@ -3157,9 +3172,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
self.to_screen(
f'{video_id}: This video is age-restricted and YouTube is requiring '
'account age-verification; some formats may be missing', only_once=True)
# tv_embedded can work around the age-verification requirement for embeddable videos
# web_creator may work around age-verification for all videos but requires PO token
append_client('tv_embedded', 'web_creator')
append_client('web_creator')
status = traverse_obj(pr, ('playabilityStatus', 'status', {str}))
if status not in ('OK', 'LIVE_STREAM_OFFLINE', 'AGE_CHECK_REQUIRED', 'AGE_VERIFICATION_REQUIRED'):
@@ -3233,7 +3247,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high', # Audio only formats
'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres',
])
skip_player_js = 'js' in self._configuration_arg('player_skip')
format_types = self._configuration_arg('formats')
skip_bad_formats = 'incomplete' not in format_types
all_formats = 'duplicate' in format_types
if self._configuration_arg('include_duplicate_formats'):
all_formats = True
@@ -3278,6 +3294,98 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
return language_code, DEFAULT_LANG_VALUE
return language_code, -1
def get_manifest_n_challenge(manifest_url):
if not url_or_none(manifest_url):
return None
# Same pattern that the player JS uses to read/replace the n challenge value
return self._search_regex(
r'/n/([^/]+)/', urllib.parse.urlparse(manifest_url).path,
'n challenge', default=None)
n_challenges = set()
s_challenges = set()
def solve_js_challenges():
# Solve all n/sig challenges in bulk and store the results in self._player_cache
challenge_requests = []
if n_challenges:
challenge_requests.append(JsChallengeRequest(
type=JsChallengeType.N,
video_id=video_id,
input=NChallengeInput(challenges=list(n_challenges), player_url=player_url)))
if s_challenges:
cached_sigfuncs = set()
for spec_id in s_challenges:
if self._load_player_data_from_cache('sigfuncs', player_url, spec_id, use_disk_cache=True):
cached_sigfuncs.add(spec_id)
s_challenges.difference_update(cached_sigfuncs)
challenge_requests.append(JsChallengeRequest(
type=JsChallengeType.SIG,
video_id=video_id,
input=SigChallengeInput(
challenges=[''.join(map(chr, range(spec_id))) for spec_id in s_challenges],
player_url=player_url)))
if challenge_requests:
for _challenge_request, challenge_response in self._jsc_director.bulk_solve(challenge_requests):
if challenge_response.type == JsChallengeType.SIG:
for challenge, result in challenge_response.output.results.items():
spec_id = len(challenge)
self._store_player_data_to_cache(
[ord(c) for c in result], 'sigfuncs',
player_url, spec_id, use_disk_cache=True)
if spec_id in s_challenges:
s_challenges.remove(spec_id)
elif challenge_response.type == JsChallengeType.N:
for challenge, result in challenge_response.output.results.items():
self._store_player_data_to_cache(result, 'n', player_url, challenge)
if challenge in n_challenges:
n_challenges.remove(challenge)
# Raise warning if any challenge requests remain
# Depending on type of challenge request
help_message = (
'Ensure you have a supported JavaScript runtime and '
'challenge solver script distribution installed. '
'Review any warnings presented before this message. '
f'For more details, refer to {_EJS_WIKI_URL}')
if s_challenges:
self.report_warning(
f'Signature solving failed: Some formats may be missing. {help_message}',
video_id=video_id, only_once=True)
if n_challenges:
self.report_warning(
f'n challenge solving failed: Some formats may be missing. {help_message}',
video_id=video_id, only_once=True)
# Clear challenge sets so that any subsequent call of this function is a no-op
s_challenges.clear()
n_challenges.clear()
# 1st pass to collect all n/sig challenges so they can later be solved at once in bulk
for streaming_data in traverse_obj(player_responses, (..., 'streamingData', {dict})):
# HTTPS formats
for fmt_stream in traverse_obj(streaming_data, (('formats', 'adaptiveFormats'), ..., {dict})):
fmt_url = fmt_stream.get('url')
s_challenge = None
if not fmt_url:
sc = urllib.parse.parse_qs(fmt_stream.get('signatureCipher'))
fmt_url = traverse_obj(sc, ('url', 0, {url_or_none}))
s_challenge = traverse_obj(sc, ('s', 0))
if s_challenge:
s_challenges.add(len(s_challenge))
if n_challenge := traverse_obj(fmt_url, ({parse_qs}, 'n', 0)):
n_challenges.add(n_challenge)
# Manifest formats
n_challenges.update(traverse_obj(
streaming_data, (('hlsManifestUrl', 'dashManifestUrl'), {get_manifest_n_challenge})))
# Final pass to extract formats and solve n/sig challenges as needed
for pr in player_responses:
streaming_data = traverse_obj(pr, 'streamingData')
if not streaming_data:
@@ -3385,10 +3493,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
def process_https_formats():
proto = 'https'
https_fmts = []
skip_player_js = 'js' in self._configuration_arg('player_skip')
for fmt_stream in streaming_formats:
if fmt_stream.get('targetDurationSec'):
# Live adaptive https formats are not supported: skip unless extractor-arg given
if fmt_stream.get('targetDurationSec') and skip_bad_formats:
continue
# FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
@@ -3422,19 +3530,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# See: https://github.com/yt-dlp/yt-dlp/issues/14883
get_language_code_and_preference(fmt_stream)
sc = urllib.parse.parse_qs(fmt_stream.get('signatureCipher'))
fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
encrypted_sig = try_get(sc, lambda x: x['s'][0])
fmt_url = traverse_obj(sc, ('url', 0, {url_or_none}))
encrypted_sig = traverse_obj(sc, ('s', 0))
if not all((sc, fmt_url, skip_player_js or player_url, encrypted_sig)):
msg = f'Some {client_name} client https formats have been skipped as they are missing a URL. '
msg_tmpl = (
'{}Some {} client https formats have been skipped as they are missing a URL. '
'{}. See https://github.com/yt-dlp/yt-dlp/issues/12482 for more details')
if client_name in ('web', 'web_safari'):
msg += 'YouTube is forcing SABR streaming for this client. '
self.write_debug(msg_tmpl.format(
f'{video_id}: ', client_name,
'YouTube is forcing SABR streaming for this client'), only_once=True)
else:
msg += (
msg = (
f'YouTube may have enabled the SABR-only streaming experiment for '
f'{"your account" if self.is_authenticated else "the current session"}. '
)
msg += 'See https://github.com/yt-dlp/yt-dlp/issues/12482 for more details'
self.report_warning(msg, video_id, only_once=True)
f'{"your account" if self.is_authenticated else "the current session"}')
self.report_warning(msg_tmpl.format('', client_name, msg), video_id, only_once=True)
continue
fmt = process_format_stream(
@@ -3444,19 +3554,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
continue
# signature
# Attempt to load sig spec from cache
if encrypted_sig:
if skip_player_js:
continue
spec_cache_id = self._sig_spec_cache_id(player_url, len(encrypted_sig))
spec = self._load_sig_spec_from_cache(spec_cache_id)
if spec:
self.write_debug(f'Using cached signature function {spec_cache_id}', only_once=True)
fmt_url += '&{}={}'.format(traverse_obj(sc, ('sp', -1)) or 'signature',
solve_sig(encrypted_sig, spec))
else:
fmt['_jsc_s_challenge'] = encrypted_sig
fmt['_jsc_s_sc'] = sc
solve_js_challenges()
spec = self._load_player_data_from_cache(
'sigfuncs', player_url, len(encrypted_sig), use_disk_cache=True)
if not spec:
continue
fmt_url += '&{}={}'.format(
traverse_obj(sc, ('sp', -1)) or 'signature',
solve_sig(encrypted_sig, spec))
# n challenge
query = parse_qs(fmt_url)
@@ -3464,10 +3572,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
if skip_player_js:
continue
n_challenge = query['n'][0]
if n_challenge in self._player_cache:
fmt_url = update_url_query(fmt_url, {'n': self._player_cache[n_challenge]})
else:
fmt['_jsc_n_challenge'] = n_challenge
solve_js_challenges()
n_result = self._load_player_data_from_cache('n', player_url, n_challenge)
if not n_result:
continue
fmt_url = update_url_query(fmt_url, {'n': n_result})
if po_token:
fmt_url = update_url_query(fmt_url, {'pot': po_token})
@@ -3484,80 +3593,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
https_fmts.append(fmt)
# Bulk process sig/n handling
# Retrieve all JSC Sig and n requests for this player response in one go
n_challenges = {}
s_challenges = {}
for fmt in https_fmts:
# This will de-duplicate requests
n_challenge = fmt.pop('_jsc_n_challenge', None)
if n_challenge is not None:
n_challenges.setdefault(n_challenge, []).append(fmt)
s_challenge = fmt.pop('_jsc_s_challenge', None)
if s_challenge is not None:
s_challenges.setdefault(len(s_challenge), {}).setdefault(s_challenge, []).append(fmt)
challenge_requests = []
if n_challenges:
challenge_requests.append(JsChallengeRequest(
type=JsChallengeType.N,
video_id=video_id,
input=NChallengeInput(challenges=list(n_challenges.keys()), player_url=player_url)))
if s_challenges:
challenge_requests.append(JsChallengeRequest(
type=JsChallengeType.SIG,
video_id=video_id,
input=SigChallengeInput(challenges=[''.join(map(chr, range(spec_id))) for spec_id in s_challenges], player_url=player_url)))
if challenge_requests:
for _challenge_request, challenge_response in self._jsc_director.bulk_solve(challenge_requests):
if challenge_response.type == JsChallengeType.SIG:
for challenge, result in challenge_response.output.results.items():
spec_id = len(challenge)
spec = [ord(c) for c in result]
self._store_sig_spec_to_cache(self._sig_spec_cache_id(player_url, spec_id), spec)
s_challenge_data = s_challenges.pop(spec_id, {})
if not s_challenge_data:
continue
for s_challenge, fmts in s_challenge_data.items():
solved_challenge = solve_sig(s_challenge, spec)
for fmt in fmts:
sc = fmt.pop('_jsc_s_sc')
fmt['url'] += '&{}={}'.format(
traverse_obj(sc, ('sp', -1)) or 'signature',
solved_challenge)
elif challenge_response.type == JsChallengeType.N:
for challenge, result in challenge_response.output.results.items():
fmts = n_challenges.pop(challenge, [])
for fmt in fmts:
self._player_cache[challenge] = result
fmt['url'] = update_url_query(fmt['url'], {'n': result})
# Raise warning if any challenge requests remain
# Depending on type of challenge request
help_message = (
'Ensure you have a supported JavaScript runtime and '
'challenge solver script distribution installed. '
'Review any warnings presented before this message. '
f'For more details, refer to {_EJS_WIKI_URL}')
if s_challenges:
self.report_warning(
f'Signature solving failed: Some formats may be missing. {help_message}',
video_id=video_id, only_once=True)
if n_challenges:
self.report_warning(
f'n challenge solving failed: Some formats may be missing. {help_message}',
video_id=video_id, only_once=True)
for cfmts in list(s_challenges.values()) + list(n_challenges.values()):
for fmt in cfmts:
if fmt in https_fmts:
https_fmts.remove(fmt)
for fmt in https_fmts:
if (all_formats or 'dashy' in format_types) and fmt['filesize']:
yield {
@@ -3573,7 +3608,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
yield from process_https_formats()
needs_live_processing = self._needs_live_processing(live_status, duration)
skip_bad_formats = 'incomplete' not in format_types
skip_manifests = set(self._configuration_arg('skip'))
if (needs_live_processing == 'is_live' # These will be filtered out by YoutubeDL anyway
@@ -3640,17 +3674,34 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
hls_manifest_url = 'hls' not in skip_manifests and streaming_data.get('hlsManifestUrl')
if hls_manifest_url:
manifest_path = urllib.parse.urlparse(hls_manifest_url).path
if m := re.fullmatch(r'(?P<path>.+)(?P<suffix>/(?:file|playlist)/index\.m3u8)', manifest_path):
manifest_path, manifest_suffix = m.group('path', 'suffix')
else:
manifest_suffix = ''
solved_n = False
n_challenge = get_manifest_n_challenge(hls_manifest_url)
if n_challenge and not skip_player_js:
solve_js_challenges()
n_result = self._load_player_data_from_cache('n', player_url, n_challenge)
if n_result:
manifest_path = manifest_path.replace(f'/n/{n_challenge}', f'/n/{n_result}')
solved_n = n_result in manifest_path
pot_policy: GvsPoTokenPolicy = self._get_default_ytcfg(
client_name)['GVS_PO_TOKEN_POLICY'][StreamingProtocol.HLS]
require_po_token = gvs_pot_required(pot_policy, is_premium_subscriber, player_token_provided)
po_token = gvs_pots.get(client_name, fetch_po_token_func(required=require_po_token or pot_policy.recommended))
if po_token:
hls_manifest_url = hls_manifest_url.rstrip('/') + f'/pot/{po_token}'
manifest_path = manifest_path.rstrip('/') + f'/pot/{po_token}'
if client_name not in gvs_pots:
gvs_pots[client_name] = po_token
if require_po_token and not po_token and 'missing_pot' not in self._configuration_arg('formats'):
self._report_pot_format_skipped(video_id, client_name, 'hls')
else:
elif solved_n or not n_challenge:
hls_manifest_url = update_url(hls_manifest_url, path=f'{manifest_path}{manifest_suffix}')
fmts, subs = self._extract_m3u8_formats_and_subtitles(
hls_manifest_url, video_id, 'mp4', fatal=False, live=live_status == 'is_live')
for sub in traverse_obj(subs, (..., ..., {dict})):
@@ -3665,17 +3716,30 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
dash_manifest_url = 'dash' not in skip_manifests and streaming_data.get('dashManifestUrl')
if dash_manifest_url:
manifest_path = urllib.parse.urlparse(dash_manifest_url).path
solved_n = False
n_challenge = get_manifest_n_challenge(dash_manifest_url)
if n_challenge and not skip_player_js:
solve_js_challenges()
n_result = self._load_player_data_from_cache('n', player_url, n_challenge)
if n_result:
manifest_path = manifest_path.replace(f'/n/{n_challenge}', f'/n/{n_result}')
solved_n = n_result in manifest_path
pot_policy: GvsPoTokenPolicy = self._get_default_ytcfg(
client_name)['GVS_PO_TOKEN_POLICY'][StreamingProtocol.DASH]
require_po_token = gvs_pot_required(pot_policy, is_premium_subscriber, player_token_provided)
po_token = gvs_pots.get(client_name, fetch_po_token_func(required=require_po_token or pot_policy.recommended))
if po_token:
dash_manifest_url = dash_manifest_url.rstrip('/') + f'/pot/{po_token}'
manifest_path = manifest_path.rstrip('/') + f'/pot/{po_token}'
if client_name not in gvs_pots:
gvs_pots[client_name] = po_token
if require_po_token and not po_token and 'missing_pot' not in self._configuration_arg('formats'):
self._report_pot_format_skipped(video_id, client_name, 'dash')
else:
elif solved_n or not n_challenge:
dash_manifest_url = update_url(dash_manifest_url, path=manifest_path)
formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
for sub in traverse_obj(subs, (..., ..., {dict})):
# TODO: If DASH video requires a PO Token, do the subs also require pot?
@@ -3683,11 +3747,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
sub[STREAMING_DATA_CLIENT_NAME] = client_name
subtitles = self._merge_subtitles(subs, subtitles) # Prioritize HLS subs over DASH
for f in formats:
if process_manifest_format(f, 'dash', client_name, f['format_id'], require_po_token and not po_token):
# Save original itag value as format_id because process_manifest_format mutates f
format_id = f['format_id']
if process_manifest_format(f, 'dash', client_name, format_id, require_po_token and not po_token):
f['filesize'] = int_or_none(self._search_regex(
r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
if needs_live_processing:
f['is_from_start'] = True
f['_itag'] = format_id
f['_client'] = client_name
yield f
yield subtitles
@@ -3800,7 +3868,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
def _download_initial_data(self, video_id, webpage, webpage_client, webpage_ytcfg):
initial_data = None
if webpage and 'initial_data' not in self._configuration_arg('webpage_skip'):
if webpage and 'initial_data' not in self._skipped_webpage_data:
initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
if not traverse_obj(initial_data, 'contents'):
self.report_warning('Incomplete data received in embedded initial data; re-fetching using API.')
@@ -3848,10 +3916,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
base_url = self.http_scheme() + '//www.youtube.com/'
webpage_url = base_url + 'watch?v=' + video_id
webpage_client = 'web'
webpage, webpage_ytcfg, initial_data, is_premium_subscriber, player_responses, player_url = self._initial_extract(
url, smuggled_data, webpage_url, webpage_client, video_id)
url, smuggled_data, webpage_url, self._webpage_client, video_id)
playability_statuses = traverse_obj(
player_responses, (..., 'playabilityStatus'), expected_type=dict)
@@ -4053,16 +4120,33 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
needs_live_processing = self._needs_live_processing(live_status, duration)
def is_bad_format(fmt):
if needs_live_processing and not fmt.get('is_from_start'):
return True
elif (live_status == 'is_live' and needs_live_processing != 'is_live'
and fmt.get('protocol') == 'http_dash_segments'):
return True
def adjust_incomplete_format(fmt, note_suffix='(Last 2 hours)', pref_adjustment=-10):
fmt['preference'] = (fmt.get('preference') or -1) + pref_adjustment
fmt['format_note'] = join_nonempty(fmt.get('format_note'), note_suffix, delim=' ')
for fmt in filter(is_bad_format, formats):
fmt['preference'] = (fmt.get('preference') or -1) - 10
fmt['format_note'] = join_nonempty(fmt.get('format_note'), '(Last 2 hours)', delim=' ')
# Adjust preference and format note for incomplete live/post-live formats
if live_status in ('is_live', 'post_live'):
for fmt in formats:
protocol = fmt.get('protocol')
# Currently, protocol isn't set for adaptive https formats, but this could change
is_adaptive = protocol in (None, 'http', 'https')
if live_status == 'post_live' and is_adaptive:
# Post-live adaptive formats cause HttpFD to raise "Did not get any data blocks"
# These formats are *only* useful to external applications, so we can hide them
# Set their preference <= -1000 so that FormatSorter flags them as 'hidden'
adjust_incomplete_format(fmt, note_suffix='(ended)', pref_adjustment=-5000)
# Is it live with --live-from-start? Or is it post-live and its duration is >2hrs?
elif needs_live_processing:
if not fmt.get('is_from_start'):
# Post-live m3u8 formats for >2hr streams
adjust_incomplete_format(fmt)
elif live_status == 'is_live':
if protocol == 'http_dash_segments':
# Live DASH formats without --live-from-start
adjust_incomplete_format(fmt)
elif is_adaptive:
# Incomplete live adaptive https formats
adjust_incomplete_format(fmt, note_suffix='(incomplete)', pref_adjustment=-20)
if needs_live_processing:
self._prepare_live_from_start_formats(
plugins/youtube_download/yt_dlp/extractor/youtube/jsc/_builtin/bun.py
+16 -1
View File
@@ -45,6 +45,8 @@ class BunJCP(EJSBaseJCP, BuiltinIEContentProvider):
JS_RUNTIME_NAME = 'bun'
BUN_NPM_LIB_FILENAME = 'yt.solver.bun.lib.js'
SUPPORTED_PROXY_SCHEMES = ['http', 'https']
_BUN_MAX_SUPPORTED_VERSION = (1, 3, 14)
_BUN_DEPRECATION_URL = 'https://github.com/yt-dlp/yt-dlp/issues/16766'
def _iter_script_sources(self):
yield from super()._iter_script_sources()
@@ -112,6 +114,19 @@ class BunJCP(EJSBaseJCP, BuiltinIEContentProvider):
return options
def _run_js_runtime(self, stdin: str, /) -> str:
is_unsupported_version = self.runtime_info.version_tuple > self._BUN_MAX_SUPPORTED_VERSION
if is_unsupported_version:
self.logger.warning(
f'bun version {".".join(map(str, self.runtime_info.version_tuple))} is not supported! '
f'{".".join(map(str, self._BUN_MAX_SUPPORTED_VERSION))} is the last supported bun version. '
f'{self.ie._downloader._format_err("DO NOT", self.ie._downloader.Styles.ERROR)} '
f'open a bug report even if you encounter any errors!',
once=True)
else:
self.logger.info(
f'bun support has been deprecated. See {self._BUN_DEPRECATION_URL} for details',
once=True)
# https://bun.com/docs/cli/run
options = ['--no-addons', '--prefer-offline']
if self._lib_script.variant == ScriptVariant.BUN_NPM:
@@ -136,7 +151,7 @@ class BunJCP(EJSBaseJCP, BuiltinIEContentProvider):
msg = f'Error running bun process (returncode: {proc.returncode})'
if stderr:
msg = f'{msg}: {stderr.strip()}'
raise JsChallengeProviderError(msg)
raise JsChallengeProviderError(msg, expected=is_unsupported_version)
return stdout
def _clean_stderr(self, stderr):
plugins/youtube_download/yt_dlp/extractor/youtube/jsc/_builtin/quickjs.py
+12 -4
View File
@@ -21,12 +21,20 @@ from yt_dlp.utils import Popen
class QuickJSJCP(EJSBaseJCP, BuiltinIEContentProvider):
PROVIDER_NAME = 'quickjs'
JS_RUNTIME_NAME = 'quickjs'
_QJS_MIN_RECOMMENDED = {
'quickjs': (2025, 4, 26),
'quickjs-ng': (0, 12, 0),
}
_QJS_WARNING_TMPL = (
'{name} versions older than {version} are missing important optimizations '
'and will solve the JS challenges very slowly. Consider upgrading.')
def _run_js_runtime(self, stdin: str, /) -> str:
if self.runtime_info.name == 'quickjs-ng':
self.logger.warning('QuickJS-NG is missing some optimizations making this very slow. Consider using upstream QuickJS instead.')
elif self.runtime_info.version_tuple < (2025, 4, 26):
self.logger.warning('Older QuickJS versions are missing optimizations making this very slow. Consider upgrading.')
min_recommended_version = self._QJS_MIN_RECOMMENDED[self.runtime_info.name]
if self.runtime_info.version_tuple < min_recommended_version:
self.logger.warning(self._QJS_WARNING_TMPL.format(
name=self.runtime_info.name,
version='.'.join(map(str, min_recommended_version))))
# QuickJS does not support reading from stdin, so we have to use a temp file
temp_file = tempfile.NamedTemporaryFile(mode='w', suffix='.js', delete=False, encoding='utf-8')
plugins/youtube_download/yt_dlp/extractor/youtube/jsc/_builtin/vendor/_info.py
Vendored
+4 -4
View File
@@ -1,10 +1,10 @@
# This file is generated by devscripts/update_ejs.py. DO NOT MODIFY!
# This file is generated by devscripts/update_requirements.py. DO NOT MODIFY!
VERSION = '0.3.2'
VERSION = '0.8.0'
HASHES = {
'yt.solver.bun.lib.js': '6ff45e94de9f0ea936a183c48173cfa9ce526ee4b7544cd556428427c1dd53c8073ef0174e79b320252bf0e7c64b0032cc1cf9c4358f3fda59033b7caa01c241',
'yt.solver.core.js': '0cd96b2d3f319dfa62cae689efa7d930ef1706e95f5921794db5089b2262957ec0a17d73938d8975ea35d0309cbfb4c8e4418d5e219837215eee242890c8b64d',
'yt.solver.core.min.js': '370d627703002b4a73b10027702734a3de9484f6b56b739942be1dc2b60fee49dee2aa86ed117d1c8ae1ac55181d326481f1fe2e2e8d5211154d48e2a55dac51',
'yt.solver.core.js': 'c163a6f376db6ce3da47d516a28a8f2a0554ae95c58dc766f0a6e2b3894f2cef1ee07fa84beb442fa471aac4f300985added1657c7c94c4d1cfefe68920ab599',
'yt.solver.core.min.js': 'ee5b307d07f55e91e4723edf5ac205cc877a474187849d757dc1322e38427b157a9d706d510c1723d3670f98e5a3f8cbcde77874a80406bd7204bc9fea30f283',
'yt.solver.deno.lib.js': '9c8ee3ab6c23e443a5a951e3ac73c6b8c1c8fb34335e7058a07bf99d349be5573611de00536dcd03ecd3cf34014c4e9b536081de37af3637c5390c6a6fd6a0f0',
'yt.solver.lib.js': '1ee3753a8222fc855f5c39db30a9ccbb7967dbe1fb810e86dc9a89aa073a0907f294c720e9b65427d560a35aa1ce6af19ef854d9126a05ca00afe03f72047733',
'yt.solver.lib.min.js': '8420c259ad16e99ce004e4651ac1bcabb53b4457bf5668a97a9359be9a998a789fee8ab124ee17f91a2ea8fd84e0f2b2fc8eabcaf0b16a186ba734cf422ad053',
plugins/youtube_download/yt_dlp/extractor/youtube/jsc/_builtin/vendor/yt.solver.core.js
Vendored
+163 -345
View File
@@ -39,189 +39,8 @@ var jsc = (function (meriyah, astring) {
function isOneOf(value, ...of) {
return of.includes(value);
}
function _optionalChain$2(ops) {
let lastAccessLHS = undefined;
let value = ops[0];
let i = 1;
while (i < ops.length) {
const op = ops[i];
const fn = ops[i + 1];
i += 2;
if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) {
return undefined;
}
if (op === 'access' || op === 'optionalAccess') {
lastAccessLHS = value;
value = fn(value);
} else if (op === 'call' || op === 'optionalCall') {
value = fn((...args) => value.call(lastAccessLHS, ...args));
lastAccessLHS = undefined;
}
}
return value;
}
const logicalExpression = {
type: 'ExpressionStatement',
expression: {
type: 'LogicalExpression',
left: { type: 'Identifier' },
right: {
type: 'SequenceExpression',
expressions: [
{
type: 'AssignmentExpression',
left: { type: 'Identifier' },
operator: '=',
right: {
type: 'CallExpression',
callee: { type: 'Identifier' },
arguments: {
or: [
[
{ type: 'Literal' },
{
type: 'CallExpression',
callee: {
type: 'Identifier',
name: 'decodeURIComponent',
},
arguments: [{ type: 'Identifier' }],
optional: false,
},
],
[
{
type: 'CallExpression',
callee: {
type: 'Identifier',
name: 'decodeURIComponent',
},
arguments: [{ type: 'Identifier' }],
optional: false,
},
],
],
},
optional: false,
},
},
{ type: 'CallExpression' },
],
},
operator: '&&',
},
};
const identifier$1 = {
or: [
{
type: 'ExpressionStatement',
expression: {
type: 'AssignmentExpression',
operator: '=',
left: { type: 'Identifier' },
right: { type: 'FunctionExpression', params: [{}, {}, {}] },
},
},
{ type: 'FunctionDeclaration', params: [{}, {}, {}] },
{
type: 'VariableDeclaration',
declarations: {
anykey: [
{
type: 'VariableDeclarator',
init: { type: 'FunctionExpression', params: [{}, {}, {}] },
},
],
},
},
],
};
function extract$1(node) {
if (!matchesStructure(node, identifier$1)) {
return null;
}
let block;
if (
node.type === 'ExpressionStatement' &&
node.expression.type === 'AssignmentExpression' &&
node.expression.right.type === 'FunctionExpression'
) {
block = node.expression.right.body;
} else if (node.type === 'VariableDeclaration') {
for (const decl of node.declarations) {
if (
decl.type === 'VariableDeclarator' &&
_optionalChain$2([
decl,
'access',
(_) => _.init,
'optionalAccess',
(_2) => _2.type,
]) === 'FunctionExpression' &&
_optionalChain$2([
decl,
'access',
(_3) => _3.init,
'optionalAccess',
(_4) => _4.params,
'access',
(_5) => _5.length,
]) === 3
) {
block = decl.init.body;
break;
}
}
} else if (node.type === 'FunctionDeclaration') {
block = node.body;
} else {
return null;
}
const relevantExpression = _optionalChain$2([
block,
'optionalAccess',
(_6) => _6.body,
'access',
(_7) => _7.at,
'call',
(_8) => _8(-2),
]);
if (!matchesStructure(relevantExpression, logicalExpression)) {
return null;
}
if (
_optionalChain$2([
relevantExpression,
'optionalAccess',
(_9) => _9.type,
]) !== 'ExpressionStatement' ||
relevantExpression.expression.type !== 'LogicalExpression' ||
relevantExpression.expression.right.type !== 'SequenceExpression' ||
relevantExpression.expression.right.expressions[0].type !==
'AssignmentExpression'
) {
return null;
}
const call = relevantExpression.expression.right.expressions[0].right;
if (call.type !== 'CallExpression' || call.callee.type !== 'Identifier') {
return null;
}
return {
type: 'ArrowFunctionExpression',
params: [{ type: 'Identifier', name: 'sig' }],
body: {
type: 'CallExpression',
callee: { type: 'Identifier', name: call.callee.name },
arguments:
call.arguments.length === 1
? [{ type: 'Identifier', name: 'sig' }]
: [call.arguments[0], { type: 'Identifier', name: 'sig' }],
optional: false,
},
async: false,
expression: false,
generator: false,
};
function generateArrowFunction(data) {
return meriyah.parse(data).body[0].expression;
}
function _optionalChain$1(ops) {
let lastAccessLHS = undefined;
@@ -246,156 +65,117 @@ var jsc = (function (meriyah, astring) {
}
const identifier = {
or: [
{
type: 'ExpressionStatement',
expression: {
type: 'AssignmentExpression',
operator: '=',
left: { or: [{ type: 'Identifier' }, { type: 'MemberExpression' }] },
right: { type: 'FunctionExpression', async: false },
},
},
{ type: 'FunctionDeclaration', async: false, id: { type: 'Identifier' } },
{
type: 'VariableDeclaration',
kind: 'var',
declarations: {
anykey: [
{
type: 'VariableDeclarator',
id: { type: 'Identifier' },
init: {
type: 'ArrayExpression',
elements: [{ type: 'Identifier' }],
},
init: { type: 'FunctionExpression', async: false },
},
],
},
},
{
type: 'ExpressionStatement',
expression: {
type: 'AssignmentExpression',
left: { type: 'Identifier' },
operator: '=',
right: {
type: 'ArrayExpression',
elements: [{ type: 'Identifier' }],
},
},
},
],
};
const catchBlockBody = [
{
type: 'ReturnStatement',
argument: {
type: 'BinaryExpression',
left: {
type: 'MemberExpression',
object: { type: 'Identifier' },
computed: true,
property: { type: 'Literal' },
optional: false,
},
right: { type: 'Identifier' },
operator: '+',
const asdasd = {
type: 'ExpressionStatement',
expression: {
type: 'CallExpression',
callee: {
type: 'MemberExpression',
object: { type: 'Identifier' },
property: {},
optional: false,
},
arguments: [
{ type: 'Literal', value: 'alr' },
{ type: 'Literal', value: 'yes' },
],
optional: false,
},
];
};
function extract(node) {
if (!matchesStructure(node, identifier)) {
let name = null;
let block = null;
switch (node.type) {
case 'ExpressionStatement': {
if (
node.expression.type === 'AssignmentExpression' &&
node.expression.left.type === 'Identifier' &&
node.expression.right.type === 'FunctionExpression' &&
node.expression.right.params.length === 1
) {
name = node.expression.left.name;
block = node.expression.right.body;
}
break;
}
case 'FunctionDeclaration': {
if (node.params.length === 1) {
name = _optionalChain$1([
node,
'access',
(_) => _.id,
'optionalAccess',
(_2) => _2.name,
]);
block = node.body;
}
break;
}
}
if (!block || !name) {
return null;
}
const tryNode = block.body.at(-2);
if (
_optionalChain$1([tryNode, 'optionalAccess', (_3) => _3.type]) !==
'TryStatement' ||
_optionalChain$1([
tryNode,
'access',
(_4) => _4.handler,
'optionalAccess',
(_5) => _5.type,
]) !== 'CatchClause'
) {
return null;
}
const catchBody = tryNode.handler.body.body;
if (matchesStructure(catchBody, catchBlockBody)) {
return makeSolverFuncFromName(name);
}
return null;
}
if (node.type === 'VariableDeclaration') {
for (const declaration of node.declarations) {
if (
declaration.type !== 'VariableDeclarator' ||
!declaration.init ||
declaration.init.type !== 'ArrayExpression' ||
declaration.init.elements.length !== 1
) {
continue;
}
const [firstElement] = declaration.init.elements;
if (firstElement && firstElement.type === 'Identifier') {
return makeSolverFuncFromName(firstElement.name);
}
const options = [];
if (node.type === 'FunctionDeclaration') {
if (
node.id &&
_optionalChain$1([
node,
'access',
(_) => _.body,
'optionalAccess',
(_2) => _2.body,
])
) {
options.push({
name: node.id,
statements: _optionalChain$1([
node,
'access',
(_3) => _3.body,
'optionalAccess',
(_4) => _4.body,
]),
});
}
} else if (node.type === 'ExpressionStatement') {
const expr = node.expression;
if (
expr.type === 'AssignmentExpression' &&
expr.left.type === 'Identifier' &&
expr.operator === '=' &&
expr.right.type === 'ArrayExpression' &&
expr.right.elements.length === 1
) {
const [firstElement] = expr.right.elements;
if (firstElement && firstElement.type === 'Identifier') {
return makeSolverFuncFromName(firstElement.name);
if (node.expression.type !== 'AssignmentExpression') {
return null;
}
const name = node.expression.left;
const body = _optionalChain$1([
node.expression.right,
'optionalAccess',
(_5) => _5.body,
'optionalAccess',
(_6) => _6.body,
]);
if (name && body) {
options.push({ name: name, statements: body });
}
} else if (node.type === 'VariableDeclaration') {
for (const declaration of node.declarations) {
const name = declaration.id;
const body = _optionalChain$1([
declaration.init,
'optionalAccess',
(_7) => _7.body,
'optionalAccess',
(_8) => _8.body,
]);
if (name && body) {
options.push({ name: name, statements: body });
}
}
}
for (const { name: name, statements: statements } of options) {
if (matchesStructure(statements, { anykey: [asdasd] })) {
return createSolver(name);
}
}
return null;
}
function makeSolverFuncFromName(name) {
return {
type: 'ArrowFunctionExpression',
params: [{ type: 'Identifier', name: 'n' }],
body: {
type: 'CallExpression',
callee: { type: 'Identifier', name: name },
arguments: [{ type: 'Identifier', name: 'n' }],
optional: false,
},
async: false,
expression: false,
generator: false,
};
function createSolver(expression) {
return generateArrowFunction(
`\n({sig, n}) => {\n const url = (${astring.generate(expression)})("https://youtube.com/watch?v=yt-dlp-wins", "s", sig ? encodeURIComponent(sig) : undefined);\n url.set("n", n);\n const proto = Object.getPrototypeOf(url);\n const keys = Object.keys(proto).concat(Object.getOwnPropertyNames(proto));\n for (const key of keys) {\n if (!["constructor", "set", "get", "clone"].includes(key)) {\n url[key]();\n break;\n }\n }\n const s = url.get("s");\n return {\n sig: s ? decodeURIComponent(s) : null,\n n: url.get("n") ?? null,\n };\n}\n`,
);
}
const setupNodes = meriyah.parse(
`\nif (typeof globalThis.XMLHttpRequest === "undefined") {\n globalThis.XMLHttpRequest = { prototype: {} };\n}\nconst window = Object.create(null);\nif (typeof URL === "undefined") {\n window.location = {\n hash: "",\n host: "www.youtube.com",\n hostname: "www.youtube.com",\n href: "https://www.youtube.com/watch?v=yt-dlp-wins",\n origin: "https://www.youtube.com",\n password: "",\n pathname: "/watch",\n port: "",\n protocol: "https:",\n search: "?v=yt-dlp-wins",\n username: "",\n };\n} else {\n window.location = new URL("https://www.youtube.com/watch?v=yt-dlp-wins");\n}\nif (typeof globalThis.document === "undefined") {\n globalThis.document = Object.create(null);\n}\nif (typeof globalThis.navigator === "undefined") {\n globalThis.navigator = Object.create(null);\n}\nif (typeof globalThis.self === "undefined") {\n globalThis.self = globalThis;\n}\n`,
`\nif (typeof globalThis.XMLHttpRequest === "undefined") {\n globalThis.XMLHttpRequest = { prototype: {} };\n}\nif (typeof URL === "undefined") {\n globalThis.location = {\n hash: "",\n host: "www.youtube.com",\n hostname: "www.youtube.com",\n href: "https://www.youtube.com/watch?v=yt-dlp-wins",\n origin: "https://www.youtube.com",\n password: "",\n pathname: "/watch",\n port: "",\n protocol: "https:",\n search: "?v=yt-dlp-wins",\n username: "",\n };\n} else {\n globalThis.location = new URL("https://www.youtube.com/watch?v=yt-dlp-wins");\n}\nif (typeof globalThis.document === "undefined") {\n globalThis.document = Object.create(null);\n}\nif (typeof globalThis.navigator === "undefined") {\n globalThis.navigator = Object.create(null);\n}\nif (typeof globalThis.self === "undefined") {\n globalThis.self = globalThis;\n}\nif (typeof globalThis.window === "undefined") {\n globalThis.window = globalThis;\n}\n`,
).body;
function _optionalChain(ops) {
let lastAccessLHS = undefined;
@@ -419,8 +199,31 @@ var jsc = (function (meriyah, astring) {
return value;
}
function preprocessPlayer(data) {
const ast = meriyah.parse(data);
const body = ast.body;
const program = meriyah.parse(data);
const plainStatements = modifyPlayer(program);
const solutions = getSolutions(plainStatements);
for (const [name, options] of Object.entries(solutions)) {
plainStatements.push({
type: 'ExpressionStatement',
expression: {
type: 'AssignmentExpression',
operator: '=',
left: {
type: 'MemberExpression',
computed: false,
object: { type: 'Identifier', name: '_result' },
property: { type: 'Identifier', name: name },
optional: false,
},
right: multiTry(options),
},
});
}
program.body.splice(0, 0, ...setupNodes);
return astring.generate(program);
}
function modifyPlayer(program) {
const body = program.body;
const block = (() => {
switch (body.length) {
case 1: {
@@ -453,16 +256,7 @@ var jsc = (function (meriyah, astring) {
}
throw 'unexpected structure';
})();
const found = { n: [], sig: [] };
const plainExpressions = block.body.filter((node) => {
const n = extract(node);
if (n) {
found.n.push(n);
}
const sig = extract$1(node);
if (sig) {
found.sig.push(sig);
}
block.body = block.body.filter((node) => {
if (node.type === 'ExpressionStatement') {
if (node.expression.type === 'AssignmentExpression') {
return true;
@@ -471,41 +265,65 @@ var jsc = (function (meriyah, astring) {
}
return true;
});
block.body = plainExpressions;
for (const [name, options] of Object.entries(found)) {
const unique = new Set(options.map((x) => JSON.stringify(x)));
if (unique.size !== 1) {
const message = `found ${unique.size} ${name} function possibilities`;
throw (
message +
(unique.size
? `: ${options.map((x) => astring.generate(x)).join(', ')}`
: '')
);
return block.body;
}
function getSolutions(statements) {
const found = { n: [], sig: [] };
for (const statement of statements) {
const result = extract(statement);
if (result) {
found.n.push(makeSolver(result, { type: 'Identifier', name: 'n' }));
found.sig.push(makeSolver(result, { type: 'Identifier', name: 'sig' }));
}
plainExpressions.push({
type: 'ExpressionStatement',
expression: {
type: 'AssignmentExpression',
operator: '=',
left: {
type: 'MemberExpression',
computed: false,
object: { type: 'Identifier', name: '_result' },
property: { type: 'Identifier', name: name },
},
right: options[0],
},
});
}
ast.body.splice(0, 0, ...setupNodes);
return astring.generate(ast);
return found;
}
function makeSolver(result, ident) {
return {
type: 'ArrowFunctionExpression',
params: [ident],
body: {
type: 'MemberExpression',
object: {
type: 'CallExpression',
callee: result,
arguments: [
{
type: 'ObjectExpression',
properties: [
{
type: 'Property',
key: ident,
value: ident,
kind: 'init',
computed: false,
method: false,
shorthand: true,
},
],
},
],
optional: false,
},
computed: false,
property: ident,
optional: false,
},
async: false,
expression: true,
generator: false,
};
}
function getFromPrepared(code) {
const resultObj = { n: null, sig: null };
Function('_result', code)(resultObj);
return resultObj;
}
function multiTry(generators) {
return generateArrowFunction(
`\n(_input) => {\n const _results = new Set();\n const errors = [];\n for (const _generator of ${astring.generate({ type: 'ArrayExpression', elements: generators })}) {\n try {\n _results.add(_generator(_input));\n } catch (e) {\n errors.push(e);\n }\n }\n if (!_results.size) {\n throw \`no solutions: \${errors.join(", ")}\`;\n }\n if (_results.size !== 1) {\n throw \`invalid solutions: \${[..._results].map(x => JSON.stringify(x)).join(", ")}\`;\n }\n return _results.values().next().value;\n}\n`,
);
}
function main(input) {
const preprocessedPlayer =
input.type === 'player'
plugins/youtube_download/yt_dlp/extractor/youtube/pot/_director.py
+8 -3
View File
@@ -6,6 +6,7 @@ import dataclasses
import datetime as dt
import hashlib
import json
import re
import traceback
import typing
import urllib.parse
@@ -63,9 +64,9 @@ class YoutubeIEContentProviderLogger(IEContentProviderLogger):
if self.log_level <= self.LogLevel.DEBUG:
self.__ie.write_debug(self._format_msg(message), only_once=once)
def info(self, message: str):
def info(self, message: str, *, once=False):
if self.log_level <= self.LogLevel.INFO:
self.__ie.to_screen(self._format_msg(message))
self.__ie.to_screen(self._format_msg(message), only_once=once)
def warning(self, message: str, *, once=False):
if self.log_level <= self.LogLevel.WARNING:
@@ -433,9 +434,13 @@ def provider_display_list(providers: Iterable[IEContentProvider]):
def clean_pot(po_token: str):
# Clean and validate the PO Token. This will strip invalid characters off
# (e.g. additional url params the user may accidentally include)
mobj = re.match(r'([^?&#]+)', urllib.parse.unquote(po_token))
if not mobj:
raise ValueError('Invalid PO Token')
try:
return base64.urlsafe_b64encode(
base64.urlsafe_b64decode(urllib.parse.unquote(po_token))).decode()
base64.urlsafe_b64decode(mobj.group(1))).decode()
except (binascii.Error, ValueError):
raise ValueError('Invalid PO Token')
plugins/youtube_download/yt_dlp/extractor/youtube/pot/_provider.py
+1 -1
View File
@@ -40,7 +40,7 @@ class IEContentProviderLogger(abc.ABC):
pass
@abc.abstractmethod
def info(self, message: str):
def info(self, message: str, *, once=False):
pass
@abc.abstractmethod
plugins/youtube_download/yt_dlp/extractor/zapiks.py
+166 -71
View File
@@ -1,110 +1,205 @@
import json
import re
import urllib.parse
from .common import InfoExtractor
from ..utils import (
clean_html,
extract_attributes,
int_or_none,
parse_duration,
parse_iso8601,
xpath_text,
xpath_with_ns,
parse_resolution,
str_or_none,
unified_timestamp,
url_or_none,
)
from ..utils.traversal import (
find_element,
find_elements,
traverse_obj,
)
class ZapiksIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?zapiks\.(?:fr|com)/(?:(?:[a-z]{2}/)?(?P<display_id>.+?)\.html|index\.php\?.*\bmedia_id=(?P<id>\d+))'
_EMBED_REGEX = [r'<iframe[^>]+src="(?P<url>https?://(?:www\.)?zapiks\.fr/index\.php\?.+?)"']
_VALID_URL = [
r'https?://(?:www\.)?zapiks\.(?:com|fr)/(?P<id>[\w-]+)\.html',
r'https?://(?:www\.)?zapiks\.fr/index\.php\?(?:[^#]+&)?media_id=(?P<id>\d+)',
]
_EMBED_REGEX = [r'<iframe\b[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?zapiks\.fr/index\.php\?(?:[^#"\']+&(?:amp;)?)?media_id=\d+)']
_TESTS = [{
'url': 'http://www.zapiks.fr/ep2s3-bon-appetit-eh-be-viva.html',
'url': 'https://www.zapiks.fr/ep2s3-bon-appetit-eh-be-viva.html',
'md5': 'aeb3c473b2d564b2d46d664d28d5f050',
'info_dict': {
'id': '80798',
'ext': 'mp4',
'title': 'EP2S3 - Bon Appétit - Eh bé viva les pyrénées con!',
'description': 'md5:7054d6f6f620c6519be1fe710d4da847',
'thumbnail': r're:https?://zpks\.com/.+\.jpg',
'description': 'md5:db07a553c1550e2905bceafa923000fd',
'display_id': 'ep2s3-bon-appetit-eh-be-viva',
'duration': 528,
'tags': 'count:5',
'thumbnail': r're:https?://zpks\.com/.+',
'timestamp': 1359044972,
'upload_date': '20130124',
'uploader': 'BonAppetit',
'uploader_id': 'bonappetit',
'view_count': int,
},
}, {
'url': 'http://www.zapiks.com/ep3s5-bon-appetit-baqueira-m-1.html',
'only_matching': True,
}, {
'url': 'http://www.zapiks.com/nl/ep3s5-bon-appetit-baqueira-m-1.html',
'only_matching': True,
}, {
'url': 'http://www.zapiks.fr/index.php?action=playerIframe&amp;media_id=118046&amp;width=640&amp;height=360&amp;autoStart=false&amp;language=fr',
'only_matching': True,
}]
_WEBPAGE_TESTS = [{
'url': 'https://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
'url': 'https://www.zapiks.com/ep3s5-bon-appetit-baqueira-m-1.html',
'md5': '196fe42901639d868956b1dcaa48de15',
'info_dict': {
'id': '118046',
'ext': 'mp4',
'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
'thumbnail': r're:https?://zpks\.com/.+\.jpg',
'display_id': 'ep3s5-bon-appetit-baqueira-m-1',
'duration': 642,
'tags': 'count:8',
'thumbnail': r're:https?://zpks\.com/.+',
'timestamp': 1424370543,
'upload_date': '20150219',
'uploader': 'BonAppetit',
'uploader_id': 'bonappetit',
'view_count': int,
},
}, {
'url': 'https://www.zapiks.fr/index.php?action=playerIframe&media_id=164049',
'md5': 'fb81a7c9b7b84c00ba111028aee593b8',
'info_dict': {
'id': '164049',
'ext': 'mp4',
'title': 'Courchevel Hiver 2025/2026',
'display_id': 'courchevel-hiver-2025-2026',
'duration': 38,
'tags': 'count:1',
'thumbnail': r're:https?://zpks\.com/.+',
'timestamp': 1769019147,
'upload_date': '20260121',
'uploader': 'jamrek',
'uploader_id': 'jamrek',
'view_count': int,
},
}, {
# https://www.youtube.com/watch?v=UBAABvegu2M
'url': 'https://www.zapiks.com/live-fwt18-vallnord-arcalis-.html',
'info_dict': {
'id': 'UBAABvegu2M',
'ext': 'mp4',
'title': 'Replay Live - FWT18 Vallnord-Arcalís Andorra - Freeride World Tour 2018',
'age_limit': 0,
'availability': 'public',
'categories': ['Sports'],
'channel': 'FIS Freeride World Tour by Peak Performance',
'channel_follower_count': int,
'channel_id': 'UCraJ3GNFfw6LXFuCV6McByg',
'channel_url': 'https://www.youtube.com/channel/UCraJ3GNFfw6LXFuCV6McByg',
'comment_count': int,
'description': 'md5:2d9fefef758d5ad0d5a987d46aff7572',
'duration': 11328,
'heatmap': 'count:100',
'like_count': int,
'live_status': 'was_live',
'media_type': 'livestream',
'playable_in_embed': True,
'release_date': '20180306',
'release_timestamp': 1520321809,
'tags': 'count:27',
'thumbnail': r're:https?://i\.ytimg\.com/.+',
'timestamp': 1520336958,
'upload_date': '20180306',
'uploader': 'FIS Freeride World Tour by Peak Performance',
'uploader_id': '@FISFreerideWorldTour',
'uploader_url': 'https://www.youtube.com/@FISFreerideWorldTour',
'view_count': int,
},
'add_ie': ['Youtube'],
}, {
# https://vimeo.com/235746460
'url': 'https://www.zapiks.fr/waking-dream-2017-full-movie.html',
'info_dict': {
'id': '235746460',
'ext': 'mp4',
'title': '"WAKING DREAM" (2017) Full Movie by Sam Favret & Julien Herry',
'duration': 1649,
'thumbnail': r're:https?://i\.vimeocdn\.com/video/.+',
'uploader': 'Favret Sam',
'uploader_id': 'samfavret',
'uploader_url': 'https://vimeo.com/samfavret',
},
'add_ie': ['Vimeo'],
'expected_warnings': ['Failed to parse XML: not well-formed'],
}]
_WEBPAGE_TESTS = [{
# https://www.zapiks.fr/ep3s5-bon-appetit-baqueira-m-1.html
# https://www.zapiks.fr/index.php?action=playerIframe&media_id=118046
'url': 'https://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
'md5': '196fe42901639d868956b1dcaa48de15',
'info_dict': {
'id': '118046',
'ext': 'mp4',
'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
'description': 'md5:b45295c3897c4c01d7c04e8484c26aaf',
'display_id': 'ep3s5-bon-appetit-baqueira-m-1',
'duration': 642,
'tags': 'count:8',
'thumbnail': r're:https?://zpks\.com/.+',
'timestamp': 1424370543,
'upload_date': '20150219',
'uploader': 'BonAppetit',
'uploader_id': 'bonappetit',
'view_count': int,
},
}]
_UPLOADER_ID_RE = re.compile(r'/pro(?:fil)?/(?P<id>[^/?#]+)/?')
def _real_extract(self, url):
mobj = self._match_valid_url(url)
video_id = mobj.group('id')
display_id = mobj.group('display_id') or video_id
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
if embed_url := traverse_obj(webpage, (
{find_element(cls='embed-container')}, {find_element(tag='iframe', html=True)},
{extract_attributes}, 'src', {self._proto_relative_url}, {url_or_none},
)):
if not self.suitable(embed_url):
return self.url_result(embed_url)
if not video_id:
video_id = self._search_regex(
r'data-media-id="(\d+)"', webpage, 'video id')
playlist = self._download_xml(
f'http://www.zapiks.fr/view/index.php?action=playlist&media_id={video_id}&lang=en',
display_id)
NS_MAP = {
'jwplayer': 'http://rss.jwpcdn.com/',
}
def ns(path):
return xpath_with_ns(path, NS_MAP)
item = playlist.find('./channel/item')
title = xpath_text(item, 'title', 'title') or self._og_search_title(webpage)
description = self._og_search_description(webpage, default=None)
thumbnail = xpath_text(
item, ns('./jwplayer:image'), 'thumbnail') or self._og_search_thumbnail(webpage, default=None)
duration = parse_duration(self._html_search_meta(
'duration', webpage, 'duration', default=None))
timestamp = parse_iso8601(self._html_search_meta(
'uploadDate', webpage, 'upload date', default=None), ' ')
view_count = int_or_none(self._search_regex(
r'UserPlays:(\d+)', webpage, 'view count', default=None))
comment_count = int_or_none(self._search_regex(
r'UserComments:(\d+)', webpage, 'comment count', default=None))
video_responsive = traverse_obj(webpage, (
{find_element(cls='video-responsive', html=True)}, {extract_attributes}, {dict}))
data_media_url = traverse_obj(video_responsive, ('data-media-url', {url_or_none}))
if data_media_url and urllib.parse.urlparse(url).path == '/index.php':
return self.url_result(data_media_url, ZapiksIE)
data_playlist = traverse_obj(video_responsive, ('data-playlist', {json.loads}, ..., any))
formats = []
for source in item.findall(ns('./jwplayer:source')):
format_id = source.attrib['label']
f = {
'url': source.attrib['file'],
for source in traverse_obj(data_playlist, (
'sources', lambda _, v: url_or_none(v['file']),
)):
format_id = traverse_obj(source, ('label', {str_or_none}))
formats.append({
'format_id': format_id,
}
m = re.search(r'^(?P<height>\d+)[pP]', format_id)
if m:
f['height'] = int(m.group('height'))
formats.append(f)
'url': source['file'],
**parse_resolution(format_id),
})
return {
'id': video_id,
'title': title,
'description': description,
'thumbnail': thumbnail,
'duration': duration,
'timestamp': timestamp,
'view_count': view_count,
'comment_count': comment_count,
'display_id': display_id,
'duration': parse_duration(self._html_search_meta('duration', webpage, default=None)),
'formats': formats,
'timestamp': unified_timestamp(self._html_search_meta('uploadDate', webpage, default=None)),
**traverse_obj(webpage, {
'description': ({find_element(cls='description-text')}, {clean_html}, filter),
'tags': (
{find_elements(cls='bs-label', html=True)},
..., {extract_attributes}, 'title', {clean_html}, filter),
'view_count': (
{find_element(cls='video-content-view-counter')}, {clean_html},
{lambda x: re.sub(r'(?:vues|views|\s+)', '', x)}, {int_or_none}),
}),
**traverse_obj(webpage, ({find_element(cls='video-content-user-link', html=True)}, {
'uploader': ({clean_html}, filter),
'uploader_id': ({extract_attributes}, 'href', {self._UPLOADER_ID_RE.fullmatch}, 'id'),
})),
**traverse_obj(data_playlist, {
'id': ('mediaid', {str_or_none}),
'title': ('title', {clean_html}, filter),
'thumbnail': ('image', {url_or_none}),
}),
}