itdominator/SolarFM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Upgrade yt_dlp and download script

Browse Source
This commit is contained in:
itdominator
2025-05-02 16:11:08 -05:00
parent 3a2e8eeb08
commit d68d9ce4f9
1194 changed files with 60099 additions and 44436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

332
plugins/youtube_download/yt_dlp/extractor/twitch.py
View File

@@ -3,14 +3,9 @@ import itertools
import json
import random
import re
import urllib.parse
from .common import InfoExtractor
from ..compat import (
compat_parse_qs,
compat_str,
compat_urllib_parse_urlencode,
compat_urllib_parse_urlparse,
)
from ..utils import (
ExtractorError,
UserNotLive,
@@ -19,19 +14,20 @@ from ..utils import (
dict_get,
float_or_none,
int_or_none,
join_nonempty,
make_archive_id,
parse_duration,
parse_iso8601,
parse_qs,
qualities,
str_or_none,
traverse_obj,
try_get,
unified_timestamp,
update_url_query,
url_or_none,
urljoin,
)
from ..utils.traversal import traverse_obj, value
class TwitchBaseIE(InfoExtractor):
@@ -47,10 +43,10 @@ class TwitchBaseIE(InfoExtractor):
'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14',
'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb',
'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777',
'ShareClipRenderStatus': 'f130048a462a0ac86bb54d653c968c514e9ab9ca94db52368c1179e97b0f16eb',
'ChannelCollectionsContent': '447aec6a0cc1e8d0a8d7732d47eb0762c336a2294fdb009e9c9d854e49d484b9',
'StreamMetadata': 'a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962',
'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
'VideoAccessToken_Clip': '36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11',
'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
'VideoMetadata': '49b5b8f268cdeb259d75b58dcb0c1a748e3b575003448a2333dc5cdafd49adad',
'VideoPlayer_ChapterSelectButtonVideo': '8d2793384aac3773beab5e59bd5d6f585aedb923d292800119e03d40cd0f9b41',
@@ -65,7 +61,7 @@ class TwitchBaseIE(InfoExtractor):
def _perform_login(self, username, password):
def fail(message):
raise ExtractorError(
'Unable to login. Twitch said: %s' % message, expected=True)
f'Unable to login. Twitch said: {message}', expected=True)
def login_step(page, urlh, note, data):
form = self._hidden_inputs(page)
@@ -157,12 +153,12 @@ class TwitchBaseIE(InfoExtractor):
'persistedQuery': {
'version': 1,
'sha256Hash': self._OPERATION_HASHES[op['operationName']],
}
},
}
return self._download_base_gql(video_id, ops, note)
def _download_access_token(self, video_id, token_kind, param_name):
method = '%sPlaybackAccessToken' % token_kind
method = f'{token_kind}PlaybackAccessToken'
ops = {
'query': '''{
%s(
@@ -177,11 +173,11 @@ class TwitchBaseIE(InfoExtractor):
value
signature
}
}''' % (method, param_name, video_id),
}''' % (method, param_name, video_id), # noqa: UP031
}
return self._download_base_gql(
video_id, ops,
'Downloading %s access token GraphQL' % token_kind)['data'][method]
f'Downloading {token_kind} access token GraphQL')['data'][method]
def _get_thumbnails(self, thumbnail):
return [{
@@ -191,6 +187,27 @@ class TwitchBaseIE(InfoExtractor):
'url': thumbnail,
}] if thumbnail else None
def _extract_twitch_m3u8_formats(self, path, video_id, token, signature):
formats = self._extract_m3u8_formats(
f'{self._USHER_BASE}/{path}/{video_id}.m3u8', video_id, 'mp4', query={
'allow_source': 'true',
'allow_audio_only': 'true',
'allow_spectre': 'true',
'p': random.randint(1000000, 10000000),
'platform': 'web',
'player': 'twitchweb',
'supported_codecs': 'av1,h265,h264',
'playlist_include_framerate': 'true',
'sig': signature,
'token': token,
})
for fmt in formats:
if fmt.get('vcodec') and fmt['vcodec'].startswith('av01'):
# mpegts does not yet have proper support for av1
fmt['downloader_options'] = {'ffmpeg_args_out': ['-f', 'mp4']}
return formats
class TwitchVodIE(TwitchBaseIE):
IE_NAME = 'twitch:vod'
@@ -222,8 +239,8 @@ class TwitchVodIE(TwitchBaseIE):
{
'start_time': 0,
'end_time': 17208,
'title': 'League of Legends'
}
'title': 'League of Legends',
},
],
'live_status': 'was_live',
},
@@ -281,25 +298,25 @@ class TwitchVodIE(TwitchBaseIE):
{
'start_time': 0,
'end_time': 573,
'title': 'League of Legends'
'title': 'League of Legends',
},
{
'start_time': 573,
'end_time': 3922,
'title': 'Legends of Runeterra'
'title': 'Legends of Runeterra',
},
{
'start_time': 3922,
'end_time': 11643,
'title': 'Art'
}
'title': 'Art',
},
],
'live_status': 'was_live',
'thumbnail': r're:^https?://.*\.jpg$',
'view_count': int,
},
'params': {
'skip_download': True
'skip_download': True,
},
}, {
'note': 'Storyboards',
@@ -318,18 +335,18 @@ class TwitchVodIE(TwitchBaseIE):
{
'start_time': 0,
'end_time': 573,
'title': 'League of Legends'
'title': 'League of Legends',
},
{
'start_time': 573,
'end_time': 3922,
'title': 'Legends of Runeterra'
'title': 'Legends of Runeterra',
},
{
'start_time': 3922,
'end_time': 11643,
'title': 'Art'
}
'title': 'Art',
},
],
'live_status': 'was_live',
'thumbnail': r're:^https?://.*\.jpg$',
@@ -339,8 +356,8 @@ class TwitchVodIE(TwitchBaseIE):
},
'params': {
'format': 'mhtml',
'skip_download': True
}
'skip_download': True,
},
}, {
'note': 'VOD with single chapter',
'url': 'https://www.twitch.tv/videos/1536751224',
@@ -357,17 +374,17 @@ class TwitchVodIE(TwitchBaseIE):
{
'start_time': 0,
'end_time': 8353,
'title': 'League of Legends'
}
'title': 'League of Legends',
},
],
'live_status': 'was_live',
'thumbnail': r're:^https?://.*\.jpg$',
'view_count': int,
},
'params': {
'skip_download': True
'skip_download': True,
},
'expected_warnings': ['Unable to download JSON metadata: HTTP Error 403: Forbidden']
'expected_warnings': ['Unable to download JSON metadata: HTTP Error 403: Forbidden'],
}, {
'url': 'https://www.twitch.tv/tangotek/schedule?vodID=1822395420',
'only_matching': True,
@@ -468,7 +485,7 @@ class TwitchVodIE(TwitchBaseIE):
vod_id = info.get('id') or item_id
# id backward compatibility for download archives
if vod_id[0] != 'v':
vod_id = 'v%s' % vod_id
vod_id = f'v{vod_id}'
thumbnail = url_or_none(info.get('previewThumbnailURL'))
is_live = None
if thumbnail:
@@ -483,8 +500,8 @@ class TwitchVodIE(TwitchBaseIE):
'description': info.get('description'),
'duration': int_or_none(info.get('lengthSeconds')),
'thumbnails': self._get_thumbnails(thumbnail),
'uploader': try_get(info, lambda x: x['owner']['displayName'], compat_str),
'uploader_id': try_get(info, lambda x: x['owner']['login'], compat_str),
'uploader': try_get(info, lambda x: x['owner']['displayName'], str),
'uploader_id': try_get(info, lambda x: x['owner']['login'], str),
'timestamp': unified_timestamp(info.get('publishedAt')),
'view_count': int_or_none(info.get('viewCount')),
'chapters': list(self._extract_chapters(info, item_id)),
@@ -532,27 +549,15 @@ class TwitchVodIE(TwitchBaseIE):
info = self._extract_info_gql(video, vod_id)
access_token = self._download_access_token(vod_id, 'video', 'id')
formats = self._extract_m3u8_formats(
'%s/vod/%s.m3u8?%s' % (
self._USHER_BASE, vod_id,
compat_urllib_parse_urlencode({
'allow_source': 'true',
'allow_audio_only': 'true',
'allow_spectre': 'true',
'player': 'twitchweb',
'playlist_include_framerate': 'true',
'nauth': access_token['value'],
'nauthsig': access_token['signature'],
})),
vod_id, 'mp4', entry_protocol='m3u8_native')
formats = self._extract_twitch_m3u8_formats(
'vod', vod_id, access_token['value'], access_token['signature'])
formats.extend(self._extract_storyboard(vod_id, video.get('storyboard'), info.get('duration')))
self._prefer_source(formats)
info['formats'] = formats
parsed_url = compat_urllib_parse_urlparse(url)
query = compat_parse_qs(parsed_url.query)
parsed_url = urllib.parse.urlparse(url)
query = urllib.parse.parse_qs(parsed_url.query)
if 't' in query:
info['start_time'] = parse_duration(query['t'][0])
@@ -560,7 +565,7 @@ class TwitchVodIE(TwitchBaseIE):
info['subtitles'] = {
'rechat': [{
'url': update_url_query(
'https://api.twitch.tv/v5/videos/%s/comments' % vod_id, {
f'https://api.twitch.tv/v5/videos/{vod_id}/comments', {
'client_id': self._CLIENT_ID,
}),
'ext': 'json',
@@ -579,7 +584,7 @@ def _make_video_result(node):
'_type': 'url_transparent',
'ie_key': TwitchVodIE.ie_key(),
'id': 'v' + video_id,
'url': 'https://www.twitch.tv/videos/%s' % video_id,
'url': f'https://www.twitch.tv/videos/{video_id}',
'title': node.get('title'),
'thumbnail': node.get('previewThumbnailURL'),
'duration': float_or_none(node.get('lengthSeconds')),
@@ -630,7 +635,7 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
def _entries(self, channel_name, *args):
cursor = None
variables_common = self._make_variables(channel_name, *args)
entries_key = '%ss' % self._ENTRY_KIND
entries_key = f'{self._ENTRY_KIND}s'
for page_num in itertools.count(1):
variables = variables_common.copy()
variables['limit'] = self._PAGE_LIMIT
@@ -641,7 +646,7 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
'operationName': self._OPERATION_NAME,
'variables': variables,
}],
'Downloading %ss GraphQL page %s' % (self._NODE_KIND, page_num),
f'Downloading {self._NODE_KIND}s GraphQL page {page_num}',
fatal=False)
if not page:
break
@@ -663,7 +668,7 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
if entry:
cursor = edge.get('cursor')
yield entry
if not cursor or not isinstance(cursor, compat_str):
if not cursor or not isinstance(cursor, str):
break
@@ -757,7 +762,7 @@ class TwitchVideosIE(TwitchPlaylistBaseIE):
if any(ie.suitable(url) for ie in (
TwitchVideosClipsIE,
TwitchVideosCollectionsIE))
else super(TwitchVideosIE, cls).suitable(url))
else super().suitable(url))
@staticmethod
def _make_variables(channel_name, broadcast_type, sort):
@@ -774,15 +779,15 @@ class TwitchVideosIE(TwitchPlaylistBaseIE):
def _real_extract(self, url):
channel_name = self._match_id(url)
qs = parse_qs(url)
filter = qs.get('filter', ['all'])[0]
video_filter = qs.get('filter', ['all'])[0]
sort = qs.get('sort', ['time'])[0]
broadcast = self._BROADCASTS.get(filter, self._DEFAULT_BROADCAST)
broadcast = self._BROADCASTS.get(video_filter, self._DEFAULT_BROADCAST)
return self.playlist_result(
self._entries(channel_name, broadcast.type, sort),
playlist_id=channel_name,
playlist_title='%s - %s sorted by %s'
% (channel_name, broadcast.label,
self._SORTED_BY.get(sort, self._DEFAULT_SORTED_BY)))
playlist_title=(
f'{channel_name} - {broadcast.label} '
f'sorted by {self._SORTED_BY.get(sort, self._DEFAULT_SORTED_BY)}'))
class TwitchVideosClipsIE(TwitchPlaylistBaseIE):
@@ -820,11 +825,11 @@ class TwitchVideosClipsIE(TwitchPlaylistBaseIE):
_NODE_KIND = 'Clip'
@staticmethod
def _make_variables(channel_name, filter):
def _make_variables(channel_name, channel_filter):
return {
'login': channel_name,
'criteria': {
'filter': filter,
'filter': channel_filter,
},
}
@@ -850,12 +855,12 @@ class TwitchVideosClipsIE(TwitchPlaylistBaseIE):
def _real_extract(self, url):
channel_name = self._match_id(url)
qs = parse_qs(url)
range = qs.get('range', ['7d'])[0]
clip = self._RANGE.get(range, self._DEFAULT_CLIP)
date_range = qs.get('range', ['7d'])[0]
clip = self._RANGE.get(date_range, self._DEFAULT_CLIP)
return self.playlist_result(
self._entries(channel_name, clip.filter),
playlist_id=channel_name,
playlist_title='%s - Clips %s' % (channel_name, clip.label))
playlist_title=f'{channel_name} - Clips {clip.label}')
class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE):
@@ -899,7 +904,7 @@ class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE):
'_type': 'url_transparent',
'ie_key': TwitchCollectionIE.ie_key(),
'id': collection_id,
'url': 'https://www.twitch.tv/collections/%s' % collection_id,
'url': f'https://www.twitch.tv/collections/{collection_id}',
'title': node.get('title'),
'thumbnail': node.get('thumbnailURL'),
'duration': float_or_none(node.get('lengthSeconds')),
@@ -911,7 +916,7 @@ class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE):
channel_name = self._match_id(url)
return self.playlist_result(
self._entries(channel_name), playlist_id=channel_name,
playlist_title='%s - Collections' % channel_name)
playlist_title=f'{channel_name} - Collections')
class TwitchStreamIE(TwitchBaseIE):
@@ -988,7 +993,7 @@ class TwitchStreamIE(TwitchBaseIE):
TwitchVideosClipsIE,
TwitchVideosCollectionsIE,
TwitchClipsIE))
else super(TwitchStreamIE, cls).suitable(url))
else super().suitable(url))
def _real_extract(self, url):
channel_name = self._match_id(url).lower()
@@ -1017,7 +1022,7 @@ class TwitchStreamIE(TwitchBaseIE):
if not user:
raise ExtractorError(
'%s does not exist' % channel_name, expected=True)
f'{channel_name} does not exist', expected=True)
stream = user['stream']
@@ -1026,23 +1031,10 @@ class TwitchStreamIE(TwitchBaseIE):
access_token = self._download_access_token(
channel_name, 'stream', 'channelName')
token = access_token['value']
stream_id = stream.get('id') or channel_name
query = {
'allow_source': 'true',
'allow_audio_only': 'true',
'allow_spectre': 'true',
'p': random.randint(1000000, 10000000),
'player': 'twitchweb',
'playlist_include_framerate': 'true',
'segment_preference': '4',
'sig': access_token['signature'].encode('utf-8'),
'token': token.encode('utf-8'),
}
formats = self._extract_m3u8_formats(
'%s/api/channel/hls/%s.m3u8' % (self._USHER_BASE, channel_name),
stream_id, 'mp4', query=query)
formats = self._extract_twitch_m3u8_formats(
'api/channel/hls', channel_name, access_token['value'], access_token['signature'])
self._prefer_source(formats)
view_count = stream.get('viewers')
@@ -1051,16 +1043,16 @@ class TwitchStreamIE(TwitchBaseIE):
sq_user = try_get(gql, lambda x: x[1]['data']['user'], dict) or {}
uploader = sq_user.get('displayName')
description = try_get(
sq_user, lambda x: x['broadcastSettings']['title'], compat_str)
sq_user, lambda x: x['broadcastSettings']['title'], str)
thumbnail = url_or_none(try_get(
gql, lambda x: x[2]['data']['user']['stream']['previewImageURL'],
compat_str))
str))
title = uploader or channel_name
stream_type = stream.get('type')
if stream_type in ['rerun', 'live']:
title += ' (%s)' % stream_type
title += f' ({stream_type})'
return {
'id': stream_id,
@@ -1092,16 +1084,44 @@ class TwitchClipsIE(TwitchBaseIE):
'url': 'https://clips.twitch.tv/FaintLightGullWholeWheat',
'md5': '761769e1eafce0ffebfb4089cb3847cd',
'info_dict': {
'id': '42850523',
'id': '396245304',
'display_id': 'FaintLightGullWholeWheat',
'ext': 'mp4',
'title': 'EA Play 2016 Live from the Novo Theatre',
'duration': 32,
'view_count': int,
'thumbnail': r're:^https?://.*\.jpg',
'timestamp': 1465767393,
'upload_date': '20160612',
'creator': 'EA',
'uploader': 'stereotype_',
'uploader_id': '43566419',
'creators': ['EA'],
'channel': 'EA',
'channel_id': '25163635',
'channel_is_verified': False,
'channel_follower_count': int,
'uploader': 'EA',
'uploader_id': '25163635',
},
}, {
'url': 'https://www.twitch.tv/xqc/clip/CulturedAmazingKuduDatSheffy-TiZ_-ixAGYR3y2Uy',
'md5': 'e90fe616b36e722a8cfa562547c543f0',
'info_dict': {
'id': '3207364882',
'display_id': 'CulturedAmazingKuduDatSheffy-TiZ_-ixAGYR3y2Uy',
'ext': 'mp4',
'title': 'A day in the life of xQc',
'duration': 60,
'view_count': int,
'thumbnail': r're:^https?://.*\.jpg',
'timestamp': 1742869615,
'upload_date': '20250325',
'creators': ['xQc'],
'channel': 'xQc',
'channel_id': '71092938',
'channel_is_verified': True,
'channel_follower_count': int,
'uploader': 'xQc',
'uploader_id': '71092938',
'categories': ['Just Chatting'],
},
}, {
# multiple formats
@@ -1125,16 +1145,14 @@ class TwitchClipsIE(TwitchBaseIE):
}]
def _real_extract(self, url):
video_id = self._match_id(url)
slug = self._match_id(url)
clip = self._download_gql(
video_id, [{
'operationName': 'VideoAccessToken_Clip',
'variables': {
'slug': video_id,
},
slug, [{
'operationName': 'ShareClipRenderStatus',
'variables': {'slug': slug},
}],
'Downloading clip access token GraphQL')[0]['data']['clip']
'Downloading clip GraphQL')[0]['data']['clip']
if not clip:
raise ExtractorError(
@@ -1144,81 +1162,71 @@ class TwitchClipsIE(TwitchBaseIE):
'sig': clip['playbackAccessToken']['signature'],
'token': clip['playbackAccessToken']['value'],
}
data = self._download_base_gql(
video_id, {
'query': '''{
clip(slug: "%s") {
broadcaster {
displayName
}
createdAt
curator {
displayName
id
}
durationSeconds
id
tiny: thumbnailURL(width: 86, height: 45)
small: thumbnailURL(width: 260, height: 147)
medium: thumbnailURL(width: 480, height: 272)
title
videoQualities {
frameRate
quality
sourceURL
}
viewCount
}
}''' % video_id}, 'Downloading clip GraphQL', fatal=False)
if data:
clip = try_get(data, lambda x: x['data']['clip'], dict) or clip
asset_default = traverse_obj(clip, ('assets', 0, {dict})) or {}
asset_portrait = traverse_obj(clip, ('assets', 1, {dict})) or {}
formats = []
for option in clip.get('videoQualities', []):
if not isinstance(option, dict):
continue
source = url_or_none(option.get('sourceURL'))
if not source:
continue
default_aspect_ratio = float_or_none(asset_default.get('aspectRatio'))
formats.extend(traverse_obj(asset_default, ('videoQualities', lambda _, v: url_or_none(v['sourceURL']), {
'url': ('sourceURL', {update_url_query(query=access_query)}),
'format_id': ('quality', {str}),
'height': ('quality', {int_or_none}),
'fps': ('frameRate', {float_or_none}),
'aspect_ratio': {value(default_aspect_ratio)},
})))
portrait_aspect_ratio = float_or_none(asset_portrait.get('aspectRatio'))
for source in traverse_obj(asset_portrait, ('videoQualities', lambda _, v: url_or_none(v['sourceURL']))):
formats.append({
'url': update_url_query(source, access_query),
'format_id': option.get('quality'),
'height': int_or_none(option.get('quality')),
'fps': int_or_none(option.get('frameRate')),
'url': update_url_query(source['sourceURL'], access_query),
'format_id': join_nonempty('portrait', source.get('quality')),
'height': int_or_none(source.get('quality')),
'fps': float_or_none(source.get('frameRate')),
'aspect_ratio': portrait_aspect_ratio,
'quality': -2,
})
thumbnails = []
for thumbnail_id in ('tiny', 'small', 'medium'):
thumbnail_url = clip.get(thumbnail_id)
if not thumbnail_url:
continue
thumb = {
'id': thumbnail_id,
'url': thumbnail_url,
}
mobj = re.search(r'-(\d+)x(\d+)\.', thumbnail_url)
if mobj:
thumb.update({
'height': int(mobj.group(2)),
'width': int(mobj.group(1)),
})
thumbnails.append(thumb)
thumb_asset_default_url = url_or_none(asset_default.get('thumbnailURL'))
if thumb_asset_default_url:
thumbnails.append({
'id': 'default',
'url': thumb_asset_default_url,
'preference': 0,
})
if thumb_asset_portrait_url := url_or_none(asset_portrait.get('thumbnailURL')):
thumbnails.append({
'id': 'portrait',
'url': thumb_asset_portrait_url,
'preference': -1,
})
thumb_default_url = url_or_none(clip.get('thumbnailURL'))
if thumb_default_url and thumb_default_url != thumb_asset_default_url:
thumbnails.append({
'id': 'small',
'url': thumb_default_url,
'preference': -2,
})
old_id = self._search_regex(r'%7C(\d+)(?:-\d+)?.mp4', formats[-1]['url'], 'old id', default=None)
return {
'id': clip.get('id') or video_id,
'id': clip.get('id') or slug,
'_old_archive_ids': [make_archive_id(self, old_id)] if old_id else None,
'display_id': video_id,
'title': clip.get('title'),
'display_id': slug,
'formats': formats,
'duration': int_or_none(clip.get('durationSeconds')),
'view_count': int_or_none(clip.get('viewCount')),
'timestamp': unified_timestamp(clip.get('createdAt')),
'thumbnails': thumbnails,
'creator': try_get(clip, lambda x: x['broadcaster']['displayName'], compat_str),
'uploader': try_get(clip, lambda x: x['curator']['displayName'], compat_str),
'uploader_id': try_get(clip, lambda x: x['curator']['id'], compat_str),
**traverse_obj(clip, {
'title': ('title', {str}),
'duration': ('durationSeconds', {int_or_none}),
'view_count': ('viewCount', {int_or_none}),
'timestamp': ('createdAt', {parse_iso8601}),
'creators': ('broadcaster', 'displayName', {str}, filter, all),
'channel': ('broadcaster', 'displayName', {str}),
'channel_id': ('broadcaster', 'id', {str}),
'channel_follower_count': ('broadcaster', 'followers', 'totalCount', {int_or_none}),
'channel_is_verified': ('broadcaster', 'isPartner', {bool}),
'uploader': ('curator', 'displayName', {str}),
'uploader_id': ('curator', 'id', {str}),
'categories': ('game', 'displayName', {str}, filter, all, filter),
}),
}