From d68d9ce4f9c8f64fd2847dae36f1ba3fe5ecefc7 Mon Sep 17 00:00:00 2001 From: itdominator <1itdominator@gmail.com> Date: Fri, 2 May 2025 16:11:08 -0500 Subject: [PATCH] Upgrade yt_dlp and download script --- plugins/youtube_download/download.sh | 29 +- plugins/youtube_download/yt_dlp/YoutubeDL.py | 785 +- plugins/youtube_download/yt_dlp/__init__.py | 186 +- plugins/youtube_download/yt_dlp/__main__.py | 2 +- .../yt_dlp/__pyinstaller/hook-yt_dlp.py | 14 +- plugins/youtube_download/yt_dlp/aes.py | 79 +- plugins/youtube_download/yt_dlp/cache.py | 4 +- plugins/youtube_download/yt_dlp/casefold.py | 5 - .../yt_dlp/compat/__init__.py | 25 +- .../yt_dlp/compat/_deprecated.py | 18 +- .../youtube_download/yt_dlp/compat/_legacy.py | 15 +- .../yt_dlp/compat/compat_utils.py | 4 +- .../yt_dlp/compat/functools.py | 26 - .../youtube_download/yt_dlp/compat/imghdr.py | 26 +- .../yt_dlp/compat/urllib/__init__.py | 2 +- .../yt_dlp/compat/urllib/request.py | 10 +- plugins/youtube_download/yt_dlp/cookies.py | 299 +- .../yt_dlp/dependencies/Cryptodome.py | 2 +- .../yt_dlp/dependencies/__init__.py | 21 +- .../yt_dlp/downloader/__init__.py | 6 +- .../yt_dlp/downloader/bunnycdn.py | 50 + .../yt_dlp/downloader/common.py | 45 +- .../yt_dlp/downloader/dash.py | 11 +- .../yt_dlp/downloader/external.py | 47 +- .../youtube_download/yt_dlp/downloader/f4m.py | 22 +- .../yt_dlp/downloader/fragment.py | 65 +- .../youtube_download/yt_dlp/downloader/hls.py | 83 +- .../yt_dlp/downloader/http.py | 29 +- .../youtube_download/yt_dlp/downloader/ism.py | 2 +- .../yt_dlp/downloader/mhtml.py | 53 +- .../yt_dlp/downloader/niconico.py | 69 +- .../yt_dlp/downloader/rtmp.py | 13 +- .../yt_dlp/downloader/rtsp.py | 4 +- .../yt_dlp/downloader/youtube_live_chat.py | 6 +- .../yt_dlp/extractor/__init__.py | 22 +- .../yt_dlp/extractor/_extractors.py | 1573 ++-- .../youtube_download/yt_dlp/extractor/abc.py | 159 +- .../yt_dlp/extractor/abcnews.py | 2 +- .../yt_dlp/extractor/abcotvs.py | 5 +- .../yt_dlp/extractor/abematv.py | 231 +- .../yt_dlp/extractor/academicearth.py | 2 +- .../yt_dlp/extractor/acast.py | 26 +- .../yt_dlp/extractor/acfun.py | 13 +- .../youtube_download/yt_dlp/extractor/adn.py | 160 +- .../yt_dlp/extractor/adobeconnect.py | 10 +- .../yt_dlp/extractor/adobepass.py | 875 +- .../yt_dlp/extractor/adobetv.py | 9 +- .../yt_dlp/extractor/adultswim.py | 7 +- .../yt_dlp/extractor/aenetworks.py | 60 +- .../yt_dlp/extractor/aeonco.py | 8 +- .../yt_dlp/extractor/afreecatv.py | 628 +- .../yt_dlp/extractor/agora.py | 31 +- .../yt_dlp/extractor/airmozilla.py | 63 - .../yt_dlp/extractor/airtv.py | 6 +- .../yt_dlp/extractor/aitube.py | 2 +- .../yt_dlp/extractor/aliexpress.py | 3 +- .../yt_dlp/extractor/aljazeera.py | 14 +- .../yt_dlp/extractor/allocine.py | 5 +- .../yt_dlp/extractor/allstar.py | 252 + .../yt_dlp/extractor/alphaporno.py | 6 +- .../yt_dlp/extractor/alsace20tv.py | 6 +- .../yt_dlp/extractor/altcensored.py | 104 + .../yt_dlp/extractor/alura.py | 30 +- .../yt_dlp/extractor/amadeustv.py | 77 + .../yt_dlp/extractor/amara.py | 12 +- .../yt_dlp/extractor/amazon.py | 8 +- .../yt_dlp/extractor/amazonminitv.py | 11 +- .../yt_dlp/extractor/amcnetworks.py | 13 +- .../yt_dlp/extractor/americastestkitchen.py | 14 +- .../youtube_download/yt_dlp/extractor/amp.py | 6 +- .../yt_dlp/extractor/anchorfm.py | 8 +- .../yt_dlp/extractor/angel.py | 12 +- .../extractor/{ant1newsgr.py => antenna.py} | 65 +- .../yt_dlp/extractor/anvato.py | 57 +- .../youtube_download/yt_dlp/extractor/aol.py | 9 +- .../youtube_download/yt_dlp/extractor/apa.py | 4 +- .../yt_dlp/extractor/appleconnect.py | 5 +- .../yt_dlp/extractor/applepodcasts.py | 80 +- .../yt_dlp/extractor/appletrailers.py | 33 +- .../yt_dlp/extractor/archiveorg.py | 379 +- .../yt_dlp/extractor/arcpublishing.py | 11 +- .../youtube_download/yt_dlp/extractor/ard.py | 710 +- .../yt_dlp/extractor/arkena.py | 6 +- .../yt_dlp/extractor/arnes.py | 14 +- .../yt_dlp/extractor/art19.py | 303 + .../youtube_download/yt_dlp/extractor/arte.py | 113 +- .../yt_dlp/extractor/asiancrush.py | 196 - .../yt_dlp/extractor/asobichannel.py | 168 + .../yt_dlp/extractor/asobistage.py | 155 + .../yt_dlp/extractor/atresplayer.py | 182 +- .../yt_dlp/extractor/atscaleconf.py | 10 +- .../yt_dlp/extractor/atttechchannel.py | 53 - .../yt_dlp/extractor/atvat.py | 26 +- .../yt_dlp/extractor/audimedia.py | 4 +- .../yt_dlp/extractor/audioboom.py | 4 +- .../yt_dlp/extractor/audiodraft.py | 13 +- .../yt_dlp/extractor/audiomack.py | 27 +- .../yt_dlp/extractor/audius.py | 46 +- .../yt_dlp/extractor/awaan.py | 27 +- .../youtube_download/yt_dlp/extractor/aws.py | 28 +- .../youtube_download/yt_dlp/extractor/axs.py | 89 + .../yt_dlp/extractor/azmedien.py | 29 +- .../yt_dlp/extractor/baidu.py | 7 +- .../yt_dlp/extractor/banbye.py | 110 +- .../yt_dlp/extractor/bandcamp.py | 84 +- .../yt_dlp/extractor/bandlab.py | 437 + .../yt_dlp/extractor/bannedvideo.py | 18 +- .../youtube_download/yt_dlp/extractor/bbc.py | 559 +- .../yt_dlp/extractor/beacon.py | 68 + .../yt_dlp/extractor/beatbump.py | 42 +- .../yt_dlp/extractor/beatport.py | 7 +- .../youtube_download/yt_dlp/extractor/beeg.py | 16 +- .../yt_dlp/extractor/behindkink.py | 3 +- .../yt_dlp/extractor/bellmedia.py | 4 +- .../yt_dlp/extractor/berufetv.py | 4 +- .../youtube_download/yt_dlp/extractor/bet.py | 11 +- .../youtube_download/yt_dlp/extractor/bfi.py | 1 + .../yt_dlp/extractor/bfmtv.py | 74 +- .../yt_dlp/extractor/bibeltv.py | 3 +- .../yt_dlp/extractor/bigflix.py | 14 +- .../youtube_download/yt_dlp/extractor/bigo.py | 5 +- .../youtube_download/yt_dlp/extractor/bild.py | 34 +- .../yt_dlp/extractor/bilibili.py | 1454 +++- .../yt_dlp/extractor/biqle.py | 110 - .../yt_dlp/extractor/bitchute.py | 69 +- .../yt_dlp/extractor/bitwave.py | 58 - .../yt_dlp/extractor/blackboardcollaborate.py | 2 +- .../yt_dlp/extractor/bleacherreport.py | 27 +- .../yt_dlp/extractor/blerp.py | 25 +- .../yt_dlp/extractor/blogger.py | 6 +- .../yt_dlp/extractor/bloomberg.py | 2 +- .../yt_dlp/extractor/bluesky.py | 393 + .../yt_dlp/extractor/bokecc.py | 17 +- .../yt_dlp/extractor/bongacams.py | 9 +- .../yt_dlp/extractor/boosty.py | 225 + .../yt_dlp/extractor/bostonglobe.py | 4 +- .../youtube_download/yt_dlp/extractor/box.py | 98 +- .../yt_dlp/extractor/boxcast.py | 16 +- .../youtube_download/yt_dlp/extractor/bpb.py | 161 +- .../youtube_download/yt_dlp/extractor/br.py | 152 +- .../yt_dlp/extractor/brainpop.py | 16 +- .../yt_dlp/extractor/bravotv.py | 11 +- .../yt_dlp/extractor/breakcom.py | 86 - .../yt_dlp/extractor/breitbart.py | 6 +- .../yt_dlp/extractor/brightcove.py | 93 +- .../yt_dlp/extractor/brilliantpala.py | 136 + .../yt_dlp/extractor/bundesliga.py | 10 +- .../yt_dlp/extractor/bundestag.py | 124 + .../yt_dlp/extractor/bunnycdn.py | 178 + .../yt_dlp/extractor/businessinsider.py | 4 +- .../yt_dlp/extractor/buzzfeed.py | 6 +- .../yt_dlp/extractor/byutv.py | 18 +- .../youtube_download/yt_dlp/extractor/c56.py | 4 +- .../yt_dlp/extractor/cableav.py | 32 - .../yt_dlp/extractor/caffeinetv.py | 74 + .../yt_dlp/extractor/callin.py | 18 +- .../yt_dlp/extractor/caltrans.py | 2 +- .../youtube_download/yt_dlp/extractor/cam4.py | 4 +- .../yt_dlp/extractor/camdemy.py | 33 +- .../yt_dlp/extractor/camfm.py | 8 +- .../yt_dlp/extractor/cammodels.py | 8 +- .../yt_dlp/extractor/camtasia.py | 6 +- .../yt_dlp/extractor/camwithher.py | 87 - .../yt_dlp/extractor/canal1.py | 39 + .../yt_dlp/extractor/canalalpha.py | 43 +- .../yt_dlp/extractor/canalc2.py | 2 +- .../yt_dlp/extractor/canalplus.py | 5 +- .../yt_dlp/extractor/canalsurmas.py | 84 + .../yt_dlp/extractor/caracoltv.py | 136 + .../yt_dlp/extractor/carambatv.py | 105 - .../yt_dlp/extractor/cartoonnetwork.py | 2 +- .../youtube_download/yt_dlp/extractor/cbc.py | 832 +- .../youtube_download/yt_dlp/extractor/cbs.py | 15 +- .../yt_dlp/extractor/cbsinteractive.py | 98 - .../yt_dlp/extractor/cbsnews.py | 3 +- .../yt_dlp/extractor/cbssports.py | 3 + .../youtube_download/yt_dlp/extractor/ccc.py | 13 +- .../youtube_download/yt_dlp/extractor/ccma.py | 73 +- .../youtube_download/yt_dlp/extractor/cctv.py | 23 +- .../youtube_download/yt_dlp/extractor/cda.py | 195 +- .../yt_dlp/extractor/cellebrite.py | 73 +- .../yt_dlp/extractor/ceskatelevize.py | 26 +- .../youtube_download/yt_dlp/extractor/cgtn.py | 28 +- .../yt_dlp/extractor/channel9.py | 252 - .../yt_dlp/extractor/chaturbate.py | 67 +- .../yt_dlp/extractor/chingari.py | 207 - .../yt_dlp/extractor/chirbit.py | 88 - .../yt_dlp/extractor/chzzk.py | 205 + .../yt_dlp/extractor/cinchcast.py | 56 - .../yt_dlp/extractor/cinemax.py | 3 +- .../yt_dlp/extractor/cinetecamilano.py | 9 +- .../yt_dlp/extractor/cineverse.py | 140 + .../yt_dlp/extractor/ciscolive.py | 4 +- .../yt_dlp/extractor/ciscowebex.py | 4 +- .../youtube_download/yt_dlp/extractor/cjsw.py | 2 +- .../yt_dlp/extractor/cliphunter.py | 76 - .../yt_dlp/extractor/clippit.py | 8 +- .../yt_dlp/extractor/cliprs.py | 3 +- .../yt_dlp/extractor/clipsyndicate.py | 52 - .../yt_dlp/extractor/closertotruth.py | 11 +- .../yt_dlp/extractor/cloudflarestream.py | 64 +- .../yt_dlp/extractor/cloudy.py | 57 - .../yt_dlp/extractor/cloudycdn.py | 98 + .../yt_dlp/extractor/clubic.py | 5 +- .../youtube_download/yt_dlp/extractor/clyp.py | 6 +- .../youtube_download/yt_dlp/extractor/cmt.py | 5 +- .../youtube_download/yt_dlp/extractor/cnbc.py | 143 +- .../youtube_download/yt_dlp/extractor/cnn.py | 315 +- .../yt_dlp/extractor/common.py | 580 +- .../yt_dlp/extractor/commonmistakes.py | 22 +- .../yt_dlp/extractor/commonprotocols.py | 2 +- .../yt_dlp/extractor/condenast.py | 35 +- .../yt_dlp/extractor/contv.py | 2 +- .../yt_dlp/extractor/corus.py | 17 +- .../youtube_download/yt_dlp/extractor/coub.py | 8 +- .../yt_dlp/extractor/cozytv.py | 10 +- .../youtube_download/yt_dlp/extractor/cpac.py | 26 +- .../yt_dlp/extractor/cracked.py | 4 +- .../yt_dlp/extractor/crackle.py | 16 +- .../yt_dlp/extractor/craftsy.py | 53 +- .../yt_dlp/extractor/crooksandliars.py | 9 +- .../yt_dlp/extractor/crowdbunker.py | 41 +- .../yt_dlp/extractor/crtvg.py | 27 +- .../yt_dlp/extractor/crunchyroll.py | 650 -- .../yt_dlp/extractor/cspan.py | 32 +- .../yt_dlp/extractor/ctsnews.py | 6 +- .../youtube_download/yt_dlp/extractor/ctv.py | 4 +- .../yt_dlp/extractor/ctvnews.py | 164 +- .../yt_dlp/extractor/cultureunplugged.py | 33 +- .../yt_dlp/extractor/curiositystream.py | 9 +- .../youtube_download/yt_dlp/extractor/cwtv.py | 101 +- .../yt_dlp/extractor/cybrary.py | 28 +- .../yt_dlp/extractor/dacast.py | 30 +- .../yt_dlp/extractor/daftsex.py | 150 - .../yt_dlp/extractor/dailymail.py | 12 +- .../yt_dlp/extractor/dailymotion.py | 315 +- .../yt_dlp/extractor/dailywire.py | 6 +- .../yt_dlp/extractor/damtomo.py | 10 +- .../yt_dlp/extractor/dangalplay.py | 197 + .../youtube_download/yt_dlp/extractor/daum.py | 38 +- .../youtube_download/yt_dlp/extractor/dbtv.py | 2 +- .../youtube_download/yt_dlp/extractor/dctp.py | 11 +- .../yt_dlp/extractor/deezer.py | 142 - .../yt_dlp/extractor/defense.py | 37 - .../yt_dlp/extractor/democracynow.py | 12 +- .../yt_dlp/extractor/detik.py | 20 +- .../yt_dlp/extractor/deuxm.py | 16 +- .../youtube_download/yt_dlp/extractor/dfb.py | 4 +- .../youtube_download/yt_dlp/extractor/dhm.py | 1 + .../youtube_download/yt_dlp/extractor/digg.py | 54 - .../yt_dlp/extractor/digitalconcerthall.py | 243 +- .../yt_dlp/extractor/digiteka.py | 2 +- .../yt_dlp/extractor/digiview.py | 130 + .../yt_dlp/extractor/discovery.py | 115 - .../yt_dlp/extractor/discoverygo.py | 172 - .../yt_dlp/extractor/disney.py | 12 +- .../yt_dlp/extractor/dispeak.py | 10 +- .../youtube_download/yt_dlp/extractor/dlf.py | 36 +- .../yt_dlp/extractor/dlive.py | 8 +- .../yt_dlp/extractor/dotsub.py | 81 - .../yt_dlp/extractor/douyutv.py | 277 +- .../yt_dlp/extractor/dplay.py | 577 +- .../yt_dlp/extractor/drbonanza.py | 2 +- .../yt_dlp/extractor/dreisat.py | 125 +- .../yt_dlp/extractor/drooble.py | 6 +- .../yt_dlp/extractor/dropbox.py | 102 +- .../yt_dlp/extractor/dropout.py | 36 +- .../yt_dlp/extractor/drtalks.py | 51 + .../yt_dlp/extractor/drtuber.py | 12 +- .../youtube_download/yt_dlp/extractor/drtv.py | 414 +- .../yt_dlp/extractor/dtube.py | 11 +- .../yt_dlp/extractor/duboku.py | 43 +- .../yt_dlp/extractor/dumpert.py | 13 +- .../yt_dlp/extractor/duoplay.py | 138 + .../youtube_download/yt_dlp/extractor/dvtv.py | 20 +- .../youtube_download/yt_dlp/extractor/dw.py | 19 +- .../yt_dlp/extractor/eagleplatform.py | 20 +- .../yt_dlp/extractor/ebaumsworld.py | 2 +- .../youtube_download/yt_dlp/extractor/ebay.py | 4 +- .../yt_dlp/extractor/echomsk.py | 43 - .../yt_dlp/extractor/egghead.py | 15 +- .../youtube_download/yt_dlp/extractor/eggs.py | 155 + .../youtube_download/yt_dlp/extractor/ehow.py | 36 - .../yt_dlp/extractor/eighttracks.py | 49 +- .../yt_dlp/extractor/einthusan.py | 105 - .../youtube_download/yt_dlp/extractor/eitb.py | 18 +- .../yt_dlp/extractor/elementorembed.py | 72 + .../yt_dlp/extractor/elevensports.py | 59 - .../yt_dlp/extractor/ellentube.py | 130 - .../yt_dlp/extractor/elpais.py | 4 +- .../yt_dlp/extractor/eltrecetv.py | 62 + .../yt_dlp/extractor/embedly.py | 2 +- .../yt_dlp/extractor/engadget.py | 15 - .../yt_dlp/extractor/epicon.py | 29 +- .../yt_dlp/extractor/epidemicsound.py | 124 + .../yt_dlp/extractor/eplus.py | 205 + .../yt_dlp/extractor/epoch.py | 10 +- .../yt_dlp/extractor/eporner.py | 26 +- .../yt_dlp/extractor/erocast.py | 63 + .../yt_dlp/extractor/eroprofile.py | 6 +- .../youtube_download/yt_dlp/extractor/err.py | 224 + .../yt_dlp/extractor/ertgr.py | 65 +- .../yt_dlp/extractor/escapist.py | 108 - .../youtube_download/yt_dlp/extractor/espn.py | 80 +- .../youtube_download/yt_dlp/extractor/esri.py | 70 - .../yt_dlp/extractor/ettutv.py | 2 +- .../yt_dlp/extractor/europa.py | 45 +- .../yt_dlp/extractor/europeantour.py | 8 +- .../yt_dlp/extractor/eurosport.py | 60 +- .../yt_dlp/extractor/euscreen.py | 21 +- .../yt_dlp/extractor/expotv.py | 74 - .../yt_dlp/extractor/expressen.py | 12 +- .../yt_dlp/extractor/extractors.py | 49 +- .../yt_dlp/extractor/extremetube.py | 48 - .../yt_dlp/extractor/eyedotv.py | 16 +- .../yt_dlp/extractor/facebook.py | 436 +- .../yt_dlp/extractor/fancode.py | 45 +- .../yt_dlp/extractor/fathom.py | 54 + .../youtube_download/yt_dlp/extractor/faz.py | 2 +- .../youtube_download/yt_dlp/extractor/fc2.py | 32 +- .../yt_dlp/extractor/fczenit.py | 2 +- .../youtube_download/yt_dlp/extractor/fifa.py | 3 +- .../yt_dlp/extractor/filmmodu.py | 69 - .../yt_dlp/extractor/filmon.py | 15 +- .../yt_dlp/extractor/filmweb.py | 2 +- .../yt_dlp/extractor/firsttv.py | 32 +- .../yt_dlp/extractor/flextv.py | 62 + .../yt_dlp/extractor/flickr.py | 14 +- .../yt_dlp/extractor/floatplane.py | 333 + .../yt_dlp/extractor/folketinget.py | 5 +- .../yt_dlp/extractor/footyroom.py | 2 +- .../yt_dlp/extractor/fourtube.py | 41 +- .../yt_dlp/extractor/fourzerostudio.py | 106 - .../youtube_download/yt_dlp/extractor/fox.py | 15 +- .../yt_dlp/extractor/foxgay.py | 58 - .../yt_dlp/extractor/fptplay.py | 2 +- .../yt_dlp/extractor/francaisfacile.py | 87 + .../yt_dlp/extractor/francetv.py | 329 +- .../yt_dlp/extractor/freesound.py | 2 +- .../yt_dlp/extractor/freetv.py | 10 +- .../yt_dlp/extractor/frontendmasters.py | 31 +- .../yt_dlp/extractor/fujitv.py | 8 +- .../yt_dlp/extractor/funimation.py | 349 - .../youtube_download/yt_dlp/extractor/funk.py | 33 +- .../yt_dlp/extractor/funker530.py | 7 +- .../yt_dlp/extractor/fusion.py | 81 - .../yt_dlp/extractor/fuyintv.py | 2 +- .../youtube_download/yt_dlp/extractor/gab.py | 25 +- .../youtube_download/yt_dlp/extractor/gaia.py | 14 +- .../yt_dlp/extractor/gamedevtv.py | 141 + .../yt_dlp/extractor/gameinformer.py | 46 - .../yt_dlp/extractor/gamejolt.py | 49 +- .../yt_dlp/extractor/gamespot.py | 5 +- .../yt_dlp/extractor/gamestar.py | 6 +- .../yt_dlp/extractor/gaskrank.py | 6 +- .../yt_dlp/extractor/gazeta.py | 5 +- .../yt_dlp/extractor/gbnews.py | 113 + .../yt_dlp/extractor/gdcvault.py | 11 +- .../yt_dlp/extractor/gedidigital.py | 4 +- .../yt_dlp/extractor/generic.py | 343 +- .../yt_dlp/extractor/genericembeds.py | 10 +- .../yt_dlp/extractor/germanupa.py | 91 + .../yt_dlp/extractor/getcourseru.py | 188 + .../yt_dlp/extractor/gettr.py | 16 +- .../yt_dlp/extractor/gfycat.py | 145 - .../yt_dlp/extractor/giantbomb.py | 2 +- .../youtube_download/yt_dlp/extractor/giga.py | 93 - .../yt_dlp/extractor/gigya.py | 20 - .../yt_dlp/extractor/glide.py | 2 +- .../yt_dlp/extractor/globalplayer.py | 4 +- .../yt_dlp/extractor/globo.py | 223 +- .../yt_dlp/extractor/glomex.py | 8 +- .../yt_dlp/extractor/gmanetwork.py | 4 +- .../youtube_download/yt_dlp/extractor/go.py | 36 +- .../yt_dlp/extractor/godresource.py | 79 + .../yt_dlp/extractor/godtube.py | 5 +- .../yt_dlp/extractor/gofile.py | 28 +- .../yt_dlp/extractor/golem.py | 18 +- .../yt_dlp/extractor/goodgame.py | 58 +- .../yt_dlp/extractor/googledrive.py | 66 +- .../yt_dlp/extractor/googlepodcasts.py | 2 +- .../yt_dlp/extractor/goplay.py | 273 +- .../yt_dlp/extractor/gopro.py | 16 +- .../yt_dlp/extractor/goshgay.py | 9 +- .../yt_dlp/extractor/gotostage.py | 28 +- .../yt_dlp/extractor/gputechconf.py | 4 +- .../yt_dlp/extractor/graspop.py | 32 + .../yt_dlp/extractor/gronkh.py | 14 +- .../yt_dlp/extractor/groupon.py | 4 +- .../yt_dlp/extractor/harpodeon.py | 16 +- .../youtube_download/yt_dlp/extractor/hbo.py | 13 +- .../yt_dlp/extractor/hearthisat.py | 46 +- .../yt_dlp/extractor/heise.py | 6 +- .../yt_dlp/extractor/helsinki.py | 38 - .../yt_dlp/extractor/hidive.py | 6 +- .../yt_dlp/extractor/historicfilms.py | 2 +- .../yt_dlp/extractor/hitbox.py | 209 - .../yt_dlp/extractor/hitrecord.py | 11 +- .../yt_dlp/extractor/hketv.py | 8 +- .../yt_dlp/extractor/hollywoodreporter.py | 2 +- .../yt_dlp/extractor/holodex.py | 2 +- .../yt_dlp/extractor/hotnewhiphop.py | 10 +- .../yt_dlp/extractor/hotstar.py | 63 +- .../yt_dlp/extractor/howcast.py | 41 - .../yt_dlp/extractor/howstuffworks.py | 86 - .../yt_dlp/extractor/hrfensehen.py | 14 +- .../youtube_download/yt_dlp/extractor/hrti.py | 22 +- .../youtube_download/yt_dlp/extractor/hse.py | 10 +- .../yt_dlp/extractor/huajiao.py | 2 +- .../yt_dlp/extractor/huffpost.py | 2 +- .../yt_dlp/extractor/hungama.py | 116 +- .../youtube_download/yt_dlp/extractor/huya.py | 97 +- .../yt_dlp/extractor/hypem.py | 6 +- .../yt_dlp/extractor/hypergryph.py | 8 +- .../yt_dlp/extractor/hytale.py | 9 +- .../yt_dlp/extractor/icareus.py | 12 +- .../yt_dlp/extractor/ichinanalive.py | 73 +- .../youtube_download/yt_dlp/extractor/ign.py | 25 +- .../yt_dlp/extractor/iheart.py | 4 +- .../yt_dlp/extractor/ilpost.py | 68 + .../yt_dlp/extractor/iltalehti.py | 2 +- .../youtube_download/yt_dlp/extractor/imdb.py | 8 +- .../yt_dlp/extractor/imggaming.py | 2 +- .../yt_dlp/extractor/imgur.py | 466 +- .../youtube_download/yt_dlp/extractor/ina.py | 2 +- .../youtube_download/yt_dlp/extractor/inc.py | 2 +- .../yt_dlp/extractor/indavideo.py | 75 +- .../yt_dlp/extractor/infoq.py | 16 +- .../yt_dlp/extractor/instagram.py | 203 +- .../yt_dlp/extractor/internazionale.py | 4 +- .../yt_dlp/extractor/iprima.py | 48 +- .../yt_dlp/extractor/iqiyi.py | 102 +- .../yt_dlp/extractor/islamchannel.py | 2 +- .../yt_dlp/extractor/israelnationalnews.py | 6 +- .../yt_dlp/extractor/itprotv.py | 21 +- .../youtube_download/yt_dlp/extractor/itv.py | 42 +- .../youtube_download/yt_dlp/extractor/ivi.py | 26 +- .../yt_dlp/extractor/ivideon.py | 16 +- .../yt_dlp/extractor/ivoox.py | 78 + .../yt_dlp/extractor/iwara.py | 11 +- .../yt_dlp/extractor/ixigua.py | 2 +- .../yt_dlp/extractor/izlesene.py | 18 +- .../yt_dlp/extractor/jable.py | 103 - .../yt_dlp/extractor/jamendo.py | 42 +- .../yt_dlp/extractor/japandiet.py | 19 +- .../yt_dlp/extractor/jeuxvideo.py | 2 + .../yt_dlp/extractor/jiocinema.py | 408 + .../yt_dlp/extractor/jiosaavn.py | 201 + .../youtube_download/yt_dlp/extractor/joj.py | 15 +- .../yt_dlp/extractor/joqrag.py | 112 + .../youtube_download/yt_dlp/extractor/jove.py | 9 +- .../yt_dlp/extractor/jstream.py | 2 +- .../youtube_download/yt_dlp/extractor/jtbc.py | 156 + .../yt_dlp/extractor/jwplatform.py | 4 +- .../yt_dlp/extractor/kakao.py | 12 +- .../yt_dlp/extractor/kaltura.py | 81 +- .../yt_dlp/extractor/kanal2.py | 66 - .../yt_dlp/extractor/kankanews.py | 7 +- .../yt_dlp/extractor/karaoketv.py | 4 +- .../yt_dlp/extractor/karrierevideos.py | 96 - .../yt_dlp/extractor/keezmovies.py | 125 - .../yt_dlp/extractor/kelbyone.py | 3 +- .../yt_dlp/extractor/kenh14.py | 160 + .../yt_dlp/extractor/khanacademy.py | 141 +- .../youtube_download/yt_dlp/extractor/kick.py | 232 +- .../yt_dlp/extractor/kicker.py | 6 +- .../youtube_download/yt_dlp/extractor/kika.py | 168 + .../yt_dlp/extractor/kinja.py | 27 +- .../yt_dlp/extractor/kommunetv.py | 12 +- .../yt_dlp/extractor/kompas.py | 2 +- .../yt_dlp/extractor/konserthusetplay.py | 119 - .../youtube_download/yt_dlp/extractor/koo.py | 28 +- .../yt_dlp/extractor/krasview.py | 1 + .../youtube_download/yt_dlp/extractor/kth.py | 7 +- .../youtube_download/yt_dlp/extractor/ku6.py | 10 +- .../yt_dlp/extractor/kukululive.py | 140 + .../youtube_download/yt_dlp/extractor/kusi.py | 83 - .../youtube_download/yt_dlp/extractor/kuwo.py | 52 +- .../youtube_download/yt_dlp/extractor/la7.py | 6 +- .../yt_dlp/extractor/laola1tv.py | 261 - .../yt_dlp/extractor/laracasts.py | 114 + .../yt_dlp/extractor/laxarxames.py | 73 + .../youtube_download/yt_dlp/extractor/lbry.py | 231 +- .../youtube_download/yt_dlp/extractor/lci.py | 27 +- .../youtube_download/yt_dlp/extractor/lcp.py | 4 +- .../yt_dlp/extractor/learningonscreen.py | 72 + .../yt_dlp/extractor/lecture2go.py | 5 +- .../yt_dlp/extractor/lecturio.py | 19 +- .../yt_dlp/extractor/leeco.py | 47 +- .../yt_dlp/extractor/lefigaro.py | 11 +- .../youtube_download/yt_dlp/extractor/lego.py | 6 +- .../yt_dlp/extractor/lenta.py | 3 +- .../yt_dlp/extractor/libraryofcongress.py | 3 +- .../yt_dlp/extractor/libsyn.py | 6 +- .../yt_dlp/extractor/lifenews.py | 29 +- .../yt_dlp/extractor/likee.py | 12 +- .../yt_dlp/extractor/limelight.py | 18 +- .../yt_dlp/extractor/linkedin.py | 196 +- .../yt_dlp/extractor/linuxacademy.py | 238 - .../yt_dlp/extractor/liputan6.py | 6 +- .../yt_dlp/extractor/listennotes.py | 28 +- .../youtube_download/yt_dlp/extractor/litv.py | 125 +- .../yt_dlp/extractor/livejournal.py | 5 +- .../yt_dlp/extractor/livestream.py | 42 +- .../yt_dlp/extractor/livestreamfails.py | 4 +- .../youtube_download/yt_dlp/extractor/lnk.py | 87 + .../yt_dlp/extractor/lnkgo.py | 163 - .../yt_dlp/extractor/localnews8.py | 42 - .../youtube_download/yt_dlp/extractor/loco.py | 159 + .../youtube_download/yt_dlp/extractor/loom.py | 466 ++ .../yt_dlp/extractor/lovehomeporn.py | 6 +- .../youtube_download/yt_dlp/extractor/lrt.py | 52 +- .../youtube_download/yt_dlp/extractor/lsm.py | 282 + .../yt_dlp/extractor/lumni.py | 9 +- .../yt_dlp/extractor/lynda.py | 54 +- .../youtube_download/yt_dlp/extractor/m6.py | 22 - .../yt_dlp/extractor/maariv.py | 62 + .../yt_dlp/extractor/magellantv.py | 52 +- .../yt_dlp/extractor/magentamusik.py | 62 + .../yt_dlp/extractor/magentamusik360.py | 58 - .../yt_dlp/extractor/mailru.py | 17 +- .../yt_dlp/extractor/mainstreaming.py | 33 +- .../yt_dlp/extractor/malltv.py | 107 - .../yt_dlp/extractor/mangomolo.py | 13 +- .../yt_dlp/extractor/manoto.py | 19 +- .../yt_dlp/extractor/manyvids.py | 177 +- .../yt_dlp/extractor/markiza.py | 11 +- .../yt_dlp/extractor/massengeschmacktv.py | 13 +- .../yt_dlp/extractor/masters.py | 3 +- .../yt_dlp/extractor/matchtv.py | 40 +- .../youtube_download/yt_dlp/extractor/mbn.py | 89 + .../youtube_download/yt_dlp/extractor/mdr.py | 58 +- .../yt_dlp/extractor/medaltv.py | 30 +- .../yt_dlp/extractor/mediaite.py | 32 +- .../yt_dlp/extractor/mediaklikk.py | 128 +- .../yt_dlp/extractor/mediaset.py | 15 +- .../yt_dlp/extractor/mediasite.py | 101 +- .../yt_dlp/extractor/mediastream.py | 26 +- .../yt_dlp/extractor/mediaworksnz.py | 10 +- .../yt_dlp/extractor/medici.py | 182 +- .../yt_dlp/extractor/megaphone.py | 10 +- .../yt_dlp/extractor/meipai.py | 4 +- .../yt_dlp/extractor/melonvod.py | 4 +- .../youtube_download/yt_dlp/extractor/meta.py | 70 - .../yt_dlp/extractor/metacafe.py | 281 - .../yt_dlp/extractor/metacritic.py | 6 +- .../yt_dlp/extractor/mgoon.py | 81 - .../youtube_download/yt_dlp/extractor/mgtv.py | 8 +- .../yt_dlp/extractor/miaopai.py | 36 - .../yt_dlp/extractor/microsoftembed.py | 305 +- .../yt_dlp/extractor/microsoftstream.py | 12 +- .../extractor/microsoftvirtualacademy.py | 189 - .../yt_dlp/extractor/mildom.py | 291 - .../yt_dlp/extractor/minds.py | 11 +- .../yt_dlp/extractor/ministrygrid.py | 55 - .../yt_dlp/extractor/minoto.py | 4 +- .../yt_dlp/extractor/miomio.py | 134 - .../yt_dlp/extractor/mirrativ.py | 6 +- .../youtube_download/yt_dlp/extractor/mit.py | 14 +- .../yt_dlp/extractor/mitele.py | 46 +- .../yt_dlp/extractor/mixch.py | 160 +- .../yt_dlp/extractor/mixcloud.py | 129 +- .../youtube_download/yt_dlp/extractor/mlb.py | 261 +- .../yt_dlp/extractor/mlssoccer.py | 69 +- .../youtube_download/yt_dlp/extractor/mnet.py | 85 - .../yt_dlp/extractor/mocha.py | 6 +- .../yt_dlp/extractor/moevideo.py | 74 - .../yt_dlp/extractor/mofosex.py | 70 - .../yt_dlp/extractor/mojevideo.py | 121 + .../yt_dlp/extractor/mojvideo.py | 6 +- .../yt_dlp/extractor/monstercat.py | 74 + .../yt_dlp/extractor/morningstar.py | 45 - .../yt_dlp/extractor/motherless.py | 41 +- .../yt_dlp/extractor/motorsport.py | 12 +- .../yt_dlp/extractor/movieclips.py | 46 - .../yt_dlp/extractor/moviepilot.py | 12 +- .../yt_dlp/extractor/moview.py | 6 +- .../yt_dlp/extractor/moviezine.py | 2 +- .../yt_dlp/extractor/movingimage.py | 4 +- .../youtube_download/yt_dlp/extractor/msn.py | 316 +- .../youtube_download/yt_dlp/extractor/mtv.py | 54 +- .../yt_dlp/extractor/muenchentv.py | 9 +- .../yt_dlp/extractor/murrtube.py | 159 +- .../yt_dlp/extractor/musescore.py | 12 +- .../yt_dlp/extractor/musicdex.py | 70 +- .../yt_dlp/extractor/mwave.py | 87 - .../youtube_download/yt_dlp/extractor/mx3.py | 171 + .../yt_dlp/extractor/mxplayer.py | 25 +- .../yt_dlp/extractor/mychannels.py | 35 - .../yt_dlp/extractor/myspace.py | 14 +- .../yt_dlp/extractor/myspass.py | 3 +- .../youtube_download/yt_dlp/extractor/myvi.py | 100 - .../yt_dlp/extractor/myvideoge.py | 2 +- .../yt_dlp/extractor/myvidster.py | 2 +- .../yt_dlp/extractor/mzaalo.py | 11 +- .../youtube_download/yt_dlp/extractor/n1.py | 108 +- .../youtube_download/yt_dlp/extractor/nate.py | 24 +- .../yt_dlp/extractor/nationalgeographic.py | 5 +- .../yt_dlp/extractor/naver.py | 210 +- .../youtube_download/yt_dlp/extractor/nba.py | 33 +- .../youtube_download/yt_dlp/extractor/nbc.py | 66 +- .../youtube_download/yt_dlp/extractor/ndr.py | 20 +- .../youtube_download/yt_dlp/extractor/ndtv.py | 27 +- .../yt_dlp/extractor/nebula.py | 650 +- .../yt_dlp/extractor/nekohacker.py | 67 +- .../yt_dlp/extractor/nerdcubed.py | 45 +- .../youtube_download/yt_dlp/extractor/nest.py | 117 + .../yt_dlp/extractor/neteasemusic.py | 708 +- .../yt_dlp/extractor/netverse.py | 14 +- .../yt_dlp/extractor/netzkino.py | 7 +- .../yt_dlp/extractor/newgrounds.py | 181 +- .../yt_dlp/extractor/newspicks.py | 2 +- .../yt_dlp/extractor/newstube.py | 75 - .../yt_dlp/extractor/newsy.py | 4 +- .../yt_dlp/extractor/nextmedia.py | 19 +- .../youtube_download/yt_dlp/extractor/nexx.py | 92 +- .../youtube_download/yt_dlp/extractor/nfb.py | 287 +- .../yt_dlp/extractor/nfhsnetwork.py | 60 +- .../youtube_download/yt_dlp/extractor/nfl.py | 347 +- .../youtube_download/yt_dlp/extractor/nhk.py | 611 +- .../youtube_download/yt_dlp/extractor/nhl.py | 12 +- .../youtube_download/yt_dlp/extractor/nick.py | 38 +- .../yt_dlp/extractor/niconico.py | 623 +- .../yt_dlp/extractor/niconicochannelplus.py | 426 + .../yt_dlp/extractor/ninaprotocol.py | 225 + .../yt_dlp/extractor/ninecninemedia.py | 16 +- .../yt_dlp/extractor/ninegag.py | 6 +- .../yt_dlp/extractor/ninenews.py | 72 + .../yt_dlp/extractor/ninenow.py | 174 +- .../yt_dlp/extractor/nintendo.py | 152 +- .../yt_dlp/extractor/nitter.py | 57 +- .../yt_dlp/extractor/njpwworld.py | 82 - .../yt_dlp/extractor/nobelprize.py | 9 +- .../yt_dlp/extractor/noice.py | 6 +- .../yt_dlp/extractor/nonktube.py | 2 +- .../yt_dlp/extractor/noodlemagazine.py | 40 +- .../yt_dlp/extractor/noovo.py | 7 +- .../yt_dlp/extractor/normalboots.py | 51 - .../yt_dlp/extractor/nosnl.py | 6 +- .../yt_dlp/extractor/nosvideo.py | 72 - .../youtube_download/yt_dlp/extractor/nova.py | 143 +- .../yt_dlp/extractor/novaplay.py | 8 +- .../yt_dlp/extractor/nowness.py | 7 +- .../youtube_download/yt_dlp/extractor/noz.py | 14 +- .../youtube_download/yt_dlp/extractor/npo.py | 42 +- .../youtube_download/yt_dlp/extractor/npr.py | 4 +- .../youtube_download/yt_dlp/extractor/nrk.py | 110 +- .../youtube_download/yt_dlp/extractor/nrl.py | 1 + .../youtube_download/yt_dlp/extractor/nts.py | 76 + .../yt_dlp/extractor/ntvde.py | 76 +- .../yt_dlp/extractor/ntvru.py | 10 +- .../yt_dlp/extractor/nubilesporn.py | 19 +- .../yt_dlp/extractor/nuevo.py | 8 +- .../youtube_download/yt_dlp/extractor/nuum.py | 201 + .../yt_dlp/extractor/nuvid.py | 10 +- .../yt_dlp/extractor/nytimes.py | 452 +- .../yt_dlp/extractor/nzherald.py | 26 +- .../yt_dlp/extractor/nzonscreen.py | 4 +- .../youtube_download/yt_dlp/extractor/nzz.py | 22 +- .../yt_dlp/extractor/odatv.py | 47 - .../yt_dlp/extractor/odkmedia.py | 6 +- .../yt_dlp/extractor/odnoklassniki.py | 19 +- .../youtube_download/yt_dlp/extractor/oftv.py | 12 +- .../yt_dlp/extractor/oktoberfesttv.py | 2 +- .../yt_dlp/extractor/olympics.py | 124 +- .../youtube_download/yt_dlp/extractor/on24.py | 31 +- .../yt_dlp/extractor/ondemandkorea.py | 198 +- .../yt_dlp/extractor/onefootball.py | 52 +- .../yt_dlp/extractor/onenewsnz.py | 16 +- .../yt_dlp/extractor/oneplace.py | 4 +- .../youtube_download/yt_dlp/extractor/onet.py | 10 +- .../yt_dlp/extractor/onionstudios.py | 3 +- .../yt_dlp/extractor/ooyala.py | 230 - .../yt_dlp/extractor/opencast.py | 4 +- .../yt_dlp/extractor/openload.py | 10 +- .../yt_dlp/extractor/openrec.py | 24 +- .../youtube_download/yt_dlp/extractor/ora.py | 9 +- .../youtube_download/yt_dlp/extractor/orf.py | 416 +- .../yt_dlp/extractor/outsidetv.py | 2 +- .../yt_dlp/extractor/packtpub.py | 12 +- .../yt_dlp/extractor/palcomp3.py | 11 +- .../yt_dlp/extractor/pandoratv.py | 128 - .../yt_dlp/extractor/panopto.py | 104 +- .../yt_dlp/extractor/paramountplus.py | 10 +- .../yt_dlp/extractor/parler.py | 8 +- .../yt_dlp/extractor/parlview.py | 9 +- .../yt_dlp/extractor/parti.py | 101 + .../yt_dlp/extractor/patreon.py | 379 +- .../youtube_download/yt_dlp/extractor/pbs.py | 95 +- .../yt_dlp/extractor/pearvideo.py | 6 +- .../yt_dlp/extractor/peekvids.py | 3 - .../yt_dlp/extractor/peertube.py | 1053 ++- .../yt_dlp/extractor/peertv.py | 2 +- .../yt_dlp/extractor/peloton.py | 26 +- .../yt_dlp/extractor/people.py | 29 - .../yt_dlp/extractor/performgroup.py | 11 +- .../yt_dlp/extractor/periscope.py | 25 +- .../yt_dlp/extractor/philharmoniedeparis.py | 7 +- .../yt_dlp/extractor/phoenix.py | 68 +- .../yt_dlp/extractor/photobucket.py | 6 +- .../yt_dlp/extractor/pialive.py | 122 + .../yt_dlp/extractor/piapro.py | 56 +- .../yt_dlp/extractor/picarto.py | 30 +- .../yt_dlp/extractor/piksel.py | 29 +- .../yt_dlp/extractor/pinkbike.py | 8 +- .../yt_dlp/extractor/pinterest.py | 38 +- .../yt_dlp/extractor/piramidetv.py | 99 + .../yt_dlp/extractor/pixivsketch.py | 9 +- .../yt_dlp/extractor/pladform.py | 17 +- .../yt_dlp/extractor/planetmarathi.py | 16 +- .../yt_dlp/extractor/platzi.py | 24 +- .../yt_dlp/extractor/playfm.py | 70 - .../yt_dlp/extractor/plays.py | 49 - .../yt_dlp/extractor/playstuff.py | 63 - .../yt_dlp/extractor/playsuisse.py | 108 +- .../yt_dlp/extractor/playtvak.py | 24 +- .../yt_dlp/extractor/playvid.py | 90 - .../yt_dlp/extractor/playwire.py | 2 +- .../yt_dlp/extractor/pluralsight.py | 71 +- .../yt_dlp/extractor/plutotv.py | 26 +- .../yt_dlp/extractor/plvideo.py | 130 + .../yt_dlp/extractor/podbayfm.py | 41 +- .../yt_dlp/extractor/podchaser.py | 16 +- .../yt_dlp/extractor/podomatic.py | 12 +- .../yt_dlp/extractor/pokemon.py | 136 - .../yt_dlp/extractor/pokergo.py | 29 +- .../yt_dlp/extractor/polsatgo.py | 12 +- .../yt_dlp/extractor/polskieradio.py | 37 +- .../yt_dlp/extractor/popcorntimes.py | 5 +- .../yt_dlp/extractor/popcorntv.py | 2 +- .../yt_dlp/extractor/porn91.py | 95 - .../yt_dlp/extractor/pornbox.py | 113 + .../yt_dlp/extractor/porncom.py | 99 - .../yt_dlp/extractor/pornez.py | 60 - .../yt_dlp/extractor/pornflip.py | 8 +- .../yt_dlp/extractor/pornhd.py | 116 - .../yt_dlp/extractor/pornhub.py | 75 +- .../yt_dlp/extractor/pornotube.py | 11 +- .../yt_dlp/extractor/pornovoisines.py | 9 +- .../yt_dlp/extractor/pornoxo.py | 3 +- .../yt_dlp/extractor/pr0gramm.py | 266 +- .../yt_dlp/extractor/prankcast.py | 91 +- .../yt_dlp/extractor/premiershiprugby.py | 2 +- .../yt_dlp/extractor/presstv.py | 10 +- .../yt_dlp/extractor/projectveritas.py | 11 +- .../yt_dlp/extractor/prosiebensat1.py | 26 +- .../youtube_download/yt_dlp/extractor/prx.py | 79 +- .../yt_dlp/extractor/puhutv.py | 43 +- .../yt_dlp/extractor/puls4.py | 3 +- .../yt_dlp/extractor/pyvideo.py | 7 +- .../yt_dlp/extractor/qdance.py | 25 +- .../yt_dlp/extractor/qingting.py | 5 +- .../yt_dlp/extractor/qqmusic.py | 551 +- .../youtube_download/yt_dlp/extractor/r7.py | 10 +- .../yt_dlp/extractor/radiko.py | 94 +- .../yt_dlp/extractor/radiobremen.py | 59 - .../yt_dlp/extractor/radiocanada.py | 10 +- .../yt_dlp/extractor/radiocomercial.py | 154 + .../yt_dlp/extractor/radiode.py | 5 +- .../yt_dlp/extractor/radiofrance.py | 385 +- .../yt_dlp/extractor/radiojavan.py | 3 +- .../yt_dlp/extractor/radiokapital.py | 18 +- .../yt_dlp/extractor/radioradicale.py | 105 + .../yt_dlp/extractor/radiozet.py | 4 +- .../yt_dlp/extractor/radlive.py | 18 +- .../youtube_download/yt_dlp/extractor/rai.py | 123 +- .../yt_dlp/extractor/raywenderlich.py | 16 +- .../yt_dlp/extractor/rbgtum.py | 89 +- .../yt_dlp/extractor/rbmaradio.py | 68 - .../youtube_download/yt_dlp/extractor/rcs.py | 30 +- .../youtube_download/yt_dlp/extractor/rcti.py | 48 +- .../youtube_download/yt_dlp/extractor/rds.py | 10 +- .../yt_dlp/extractor/recurbate.py | 42 - .../yt_dlp/extractor/redbee.py | 31 +- .../yt_dlp/extractor/redbulltv.py | 21 +- .../yt_dlp/extractor/reddit.py | 181 +- .../yt_dlp/extractor/redge.py | 134 + .../yt_dlp/extractor/redgifs.py | 76 +- .../yt_dlp/extractor/redtube.py | 16 +- .../yt_dlp/extractor/regiotv.py | 55 - .../yt_dlp/extractor/rentv.py | 9 +- .../yt_dlp/extractor/restudy.py | 5 +- .../yt_dlp/extractor/reuters.py | 11 +- .../yt_dlp/extractor/reverbnation.py | 8 +- .../youtube_download/yt_dlp/extractor/rice.py | 112 - .../yt_dlp/extractor/ridehome.py | 96 + .../yt_dlp/extractor/rinsefm.py | 89 + .../yt_dlp/extractor/rmcdecouverte.py | 10 +- .../yt_dlp/extractor/rockstargames.py | 11 +- .../yt_dlp/extractor/rokfin.py | 36 +- .../yt_dlp/extractor/roosterteeth.py | 205 +- .../yt_dlp/extractor/rottentomatoes.py | 4 +- .../youtube_download/yt_dlp/extractor/roya.py | 43 + .../yt_dlp/extractor/rozhlas.py | 44 +- .../youtube_download/yt_dlp/extractor/rte.py | 4 +- .../youtube_download/yt_dlp/extractor/rtl2.py | 104 +- .../yt_dlp/extractor/rtlnl.py | 26 +- .../yt_dlp/extractor/rtnews.py | 60 +- .../youtube_download/yt_dlp/extractor/rtp.py | 231 +- .../yt_dlp/extractor/rtrfm.py | 4 +- .../youtube_download/yt_dlp/extractor/rts.py | 12 +- .../yt_dlp/extractor/rtvcplay.py | 13 +- .../youtube_download/yt_dlp/extractor/rtve.py | 435 +- .../yt_dlp/extractor/rtvnh.py | 58 - .../youtube_download/yt_dlp/extractor/rtvs.py | 22 +- .../yt_dlp/extractor/rtvslo.py | 187 +- .../yt_dlp/extractor/rudovideo.py | 135 + .../youtube_download/yt_dlp/extractor/ruhd.py | 42 - .../yt_dlp/extractor/rule34video.py | 90 +- .../yt_dlp/extractor/rumble.py | 152 +- .../yt_dlp/extractor/rutube.py | 244 +- .../youtube_download/yt_dlp/extractor/rutv.py | 16 +- .../yt_dlp/extractor/ruutu.py | 18 +- .../youtube_download/yt_dlp/extractor/ruv.py | 8 +- .../youtube_download/yt_dlp/extractor/s4c.py | 55 +- .../yt_dlp/extractor/safari.py | 30 +- .../yt_dlp/extractor/saitosan.py | 9 +- .../yt_dlp/extractor/samplefocus.py | 20 +- .../youtube_download/yt_dlp/extractor/sapo.py | 2 +- .../yt_dlp/extractor/savefrom.py | 30 - .../youtube_download/yt_dlp/extractor/sbs.py | 13 +- .../yt_dlp/extractor/sbscokr.py | 200 + .../yt_dlp/extractor/screencast.py | 15 +- .../yt_dlp/extractor/screencastify.py | 20 +- .../yt_dlp/extractor/screencastomatic.py | 2 +- .../yt_dlp/extractor/screenrec.py | 33 + .../yt_dlp/extractor/scrippsnetworks.py | 22 +- .../yt_dlp/extractor/scrolller.py | 14 +- .../youtube_download/yt_dlp/extractor/scte.py | 10 +- .../yt_dlp/extractor/seeker.py | 55 - .../yt_dlp/extractor/sejmpl.py | 218 + .../youtube_download/yt_dlp/extractor/sen.py | 36 + .../yt_dlp/extractor/senalcolombia.py | 1 + .../yt_dlp/extractor/senategov.py | 234 +- .../yt_dlp/extractor/sendtonews.py | 15 +- .../yt_dlp/extractor/servus.py | 31 +- .../yt_dlp/extractor/sevenplus.py | 7 +- .../youtube_download/yt_dlp/extractor/sexu.py | 3 +- .../yt_dlp/extractor/seznamzpravy.py | 14 +- .../yt_dlp/extractor/shahid.py | 20 +- .../yt_dlp/extractor/shared.py | 138 - .../yt_dlp/extractor/sharepoint.py | 112 + .../yt_dlp/extractor/shemaroome.py | 29 +- .../yt_dlp/extractor/showroomlive.py | 9 +- .../yt_dlp/extractor/sibnet.py | 4 +- .../yt_dlp/extractor/simplecast.py | 6 +- .../youtube_download/yt_dlp/extractor/sina.py | 9 +- .../yt_dlp/extractor/sixplay.py | 13 +- .../youtube_download/yt_dlp/extractor/skeb.py | 26 +- .../youtube_download/yt_dlp/extractor/sky.py | 35 +- .../yt_dlp/extractor/skyit.py | 109 +- .../yt_dlp/extractor/skylinewebcams.py | 3 +- .../yt_dlp/extractor/skynewsarabia.py | 15 +- .../yt_dlp/extractor/skynewsau.py | 12 +- .../yt_dlp/extractor/slideshare.py | 8 +- .../yt_dlp/extractor/slideslive.py | 117 +- .../yt_dlp/extractor/slutload.py | 12 +- .../yt_dlp/extractor/snapchat.py | 76 + .../yt_dlp/extractor/snotr.py | 2 +- .../yt_dlp/extractor/softwhiteunderbelly.py | 87 + .../youtube_download/yt_dlp/extractor/sohu.py | 153 +- .../yt_dlp/extractor/sonyliv.py | 84 +- .../yt_dlp/extractor/soundcloud.py | 498 +- .../yt_dlp/extractor/soundgasm.py | 4 +- .../yt_dlp/extractor/southpark.py | 4 +- .../yt_dlp/extractor/sovietscloset.py | 49 +- .../yt_dlp/extractor/spankbang.py | 17 +- .../yt_dlp/extractor/spankwire.py | 174 - .../yt_dlp/extractor/spiegel.py | 4 +- .../yt_dlp/extractor/sport5.py | 6 +- .../yt_dlp/extractor/sportdeutschland.py | 12 +- .../yt_dlp/extractor/spotify.py | 6 +- .../yt_dlp/extractor/spreaker.py | 126 +- .../yt_dlp/extractor/springboardplatform.py | 11 +- .../yt_dlp/extractor/sproutvideo.py | 198 + .../yt_dlp/extractor/srgssr.py | 15 +- .../yt_dlp/extractor/srmediathek.py | 3 +- .../yt_dlp/extractor/stacommu.py | 164 +- .../yt_dlp/extractor/stageplus.py | 23 +- .../yt_dlp/extractor/stanfordoc.py | 18 +- .../yt_dlp/extractor/startrek.py | 5 +- .../yt_dlp/extractor/startv.py | 35 +- .../yt_dlp/extractor/steam.py | 42 +- .../yt_dlp/extractor/stitcher.py | 7 +- .../yt_dlp/extractor/storyfire.py | 14 +- .../yt_dlp/extractor/streaks.py | 236 + .../yt_dlp/extractor/streamable.py | 12 +- .../yt_dlp/extractor/streamcloud.py | 75 - .../yt_dlp/extractor/streamcz.py | 14 +- .../yt_dlp/extractor/streamff.py | 30 - .../yt_dlp/extractor/streetvoice.py | 8 +- .../yt_dlp/extractor/stretchinternet.py | 2 +- .../yt_dlp/extractor/stripchat.py | 29 +- .../youtube_download/yt_dlp/extractor/stv.py | 11 +- .../yt_dlp/extractor/subsplash.py | 199 + .../yt_dlp/extractor/substack.py | 53 +- .../yt_dlp/extractor/sunporno.py | 10 +- .../yt_dlp/extractor/sverigesradio.py | 2 +- .../youtube_download/yt_dlp/extractor/svt.py | 163 +- .../yt_dlp/extractor/swearnet.py | 72 +- .../yt_dlp/extractor/swrmediathek.py | 111 - .../youtube_download/yt_dlp/extractor/syfy.py | 8 +- .../yt_dlp/extractor/syvdk.py | 4 +- .../yt_dlp/extractor/tagesschau.py | 5 +- .../yt_dlp/extractor/taptap.py | 275 + .../youtube_download/yt_dlp/extractor/tass.py | 1 + .../youtube_download/yt_dlp/extractor/tbs.py | 19 +- .../yt_dlp/extractor/tbsjp.py | 150 + .../yt_dlp/extractor/tdslifeway.py | 31 - .../yt_dlp/extractor/teachable.py | 40 +- .../yt_dlp/extractor/teachertube.py | 16 +- .../yt_dlp/extractor/teachingchannel.py | 1 + .../yt_dlp/extractor/teamcoco.py | 4 +- .../yt_dlp/extractor/teamtreehouse.py | 2 +- .../yt_dlp/extractor/techtalks.py | 80 - .../youtube_download/yt_dlp/extractor/ted.py | 27 +- .../yt_dlp/extractor/tele13.py | 6 +- .../yt_dlp/extractor/tele5.py | 133 +- .../yt_dlp/extractor/telecaribe.py | 2 +- .../yt_dlp/extractor/telecinco.py | 143 +- .../yt_dlp/extractor/telegraaf.py | 8 +- .../yt_dlp/extractor/telegram.py | 2 +- .../yt_dlp/extractor/telemb.py | 7 +- .../yt_dlp/extractor/telemundo.py | 6 +- .../yt_dlp/extractor/telequebec.py | 9 +- .../yt_dlp/extractor/teletask.py | 9 +- .../yt_dlp/extractor/telewebion.py | 158 +- .../yt_dlp/extractor/tempo.py | 20 +- .../yt_dlp/extractor/tencent.py | 9 +- .../yt_dlp/extractor/tennistv.py | 14 +- .../yt_dlp/extractor/tenplay.py | 162 +- .../yt_dlp/extractor/testurl.py | 2 +- .../youtube_download/yt_dlp/extractor/tf1.py | 21 +- .../youtube_download/yt_dlp/extractor/tfo.py | 4 +- .../yt_dlp/extractor/theguardian.py | 134 + .../yt_dlp/extractor/theholetv.py | 6 +- .../yt_dlp/extractor/theintercept.py | 11 +- .../yt_dlp/extractor/theplatform.py | 73 +- .../yt_dlp/extractor/thestar.py | 2 +- .../yt_dlp/extractor/theta.py | 90 - .../yt_dlp/extractor/theweatherchannel.py | 28 +- .../yt_dlp/extractor/thisamericanlife.py | 4 +- .../yt_dlp/extractor/thisav.py | 66 - .../yt_dlp/extractor/thisoldhouse.py | 117 +- .../yt_dlp/extractor/thisvid.py | 6 +- .../yt_dlp/extractor/threeqsdn.py | 8 +- .../yt_dlp/extractor/threespeak.py | 28 +- .../yt_dlp/extractor/tiktok.py | 998 ++- .../yt_dlp/extractor/tinypic.py | 54 - .../youtube_download/yt_dlp/extractor/tmz.py | 270 +- .../yt_dlp/extractor/tnaflix.py | 14 +- .../yt_dlp/extractor/toggle.py | 48 +- .../yt_dlp/extractor/tokentube.py | 153 - .../yt_dlp/extractor/tonline.py | 6 +- .../yt_dlp/extractor/toongoggles.py | 2 +- .../yt_dlp/extractor/toutv.py | 2 +- .../yt_dlp/extractor/toypics.py | 13 +- .../yt_dlp/extractor/traileraddict.py | 4 +- .../yt_dlp/extractor/triller.py | 2 +- .../yt_dlp/extractor/trilulilu.py | 100 - .../yt_dlp/extractor/trovo.py | 4 +- .../yt_dlp/extractor/trtcocuk.py | 6 +- .../yt_dlp/extractor/trtworld.py | 101 + .../yt_dlp/extractor/trueid.py | 10 +- .../yt_dlp/extractor/truth.py | 1 - .../yt_dlp/extractor/trutv.py | 2 +- .../yt_dlp/extractor/tube8.py | 99 +- .../yt_dlp/extractor/tubetugraz.py | 79 +- .../yt_dlp/extractor/tubitv.py | 163 +- .../yt_dlp/extractor/tumblr.py | 401 +- .../yt_dlp/extractor/tunein.py | 4 +- .../yt_dlp/extractor/tunepk.py | 87 - .../yt_dlp/extractor/turbo.py | 64 - .../yt_dlp/extractor/turner.py | 25 +- .../youtube_download/yt_dlp/extractor/tv2.py | 24 +- .../yt_dlp/extractor/tv24ua.py | 6 +- .../yt_dlp/extractor/tv2dk.py | 76 +- .../yt_dlp/extractor/tv2hu.py | 24 +- .../youtube_download/yt_dlp/extractor/tv4.py | 2 +- .../yt_dlp/extractor/tv5mondeplus.py | 207 +- .../yt_dlp/extractor/tv5unis.py | 12 +- .../youtube_download/yt_dlp/extractor/tva.py | 105 +- .../yt_dlp/extractor/tvanouvelles.py | 6 +- .../youtube_download/yt_dlp/extractor/tvc.py | 2 +- .../youtube_download/yt_dlp/extractor/tver.py | 235 +- .../yt_dlp/extractor/tvigle.py | 8 +- .../yt_dlp/extractor/tviplayer.py | 8 +- .../yt_dlp/extractor/tvn24.py | 9 +- .../yt_dlp/extractor/tvnet.py | 138 - .../yt_dlp/extractor/tvnoe.py | 5 +- .../yt_dlp/extractor/tvnow.py | 639 -- .../youtube_download/yt_dlp/extractor/tvp.py | 39 +- .../yt_dlp/extractor/tvplay.py | 16 +- .../yt_dlp/extractor/tvplayer.py | 9 +- .../youtube_download/yt_dlp/extractor/tvw.py | 165 + .../yt_dlp/extractor/tweakers.py | 6 +- .../yt_dlp/extractor/twentyfourvideo.py | 128 - .../yt_dlp/extractor/twentymin.py | 6 +- .../yt_dlp/extractor/twentythreevideo.py | 4 +- .../yt_dlp/extractor/twitcasting.py | 128 +- .../yt_dlp/extractor/twitch.py | 332 +- .../yt_dlp/extractor/twitter.py | 694 +- .../youtube_download/yt_dlp/extractor/txxx.py | 58 +- .../yt_dlp/extractor/udemy.py | 44 +- .../youtube_download/yt_dlp/extractor/udn.py | 8 +- .../yt_dlp/extractor/ukcolumn.py | 11 +- .../yt_dlp/extractor/uktvplay.py | 2 +- .../yt_dlp/extractor/uliza.py | 113 + .../youtube_download/yt_dlp/extractor/umg.py | 5 +- .../yt_dlp/extractor/unistra.py | 8 +- .../yt_dlp/extractor/unity.py | 3 +- .../yt_dlp/extractor/unscripted.py | 53 - .../yt_dlp/extractor/unsupported.py | 47 +- .../youtube_download/yt_dlp/extractor/uol.py | 16 +- .../yt_dlp/extractor/urort.py | 13 +- .../yt_dlp/extractor/urplay.py | 12 +- .../yt_dlp/extractor/usatoday.py | 7 +- .../yt_dlp/extractor/ustream.py | 39 +- .../yt_dlp/extractor/ustudio.py | 14 +- .../yt_dlp/extractor/utreon.py | 43 +- .../yt_dlp/extractor/varzesh3.py | 1 + .../yt_dlp/extractor/vbox7.py | 82 +- .../yt_dlp/extractor/veehd.py | 116 - .../youtube_download/yt_dlp/extractor/veo.py | 7 +- .../youtube_download/yt_dlp/extractor/veoh.py | 188 - .../yt_dlp/extractor/vesti.py | 7 +- .../youtube_download/yt_dlp/extractor/vevo.py | 45 +- .../youtube_download/yt_dlp/extractor/vgtv.py | 19 +- .../youtube_download/yt_dlp/extractor/vh1.py | 2 +- .../youtube_download/yt_dlp/extractor/vice.py | 28 +- .../yt_dlp/extractor/vidbit.py | 82 - .../yt_dlp/extractor/viddler.py | 6 +- .../yt_dlp/extractor/videa.py | 12 +- .../yt_dlp/extractor/videocampus_sachsen.py | 113 +- .../yt_dlp/extractor/videofyme.py | 7 +- .../yt_dlp/extractor/videoken.py | 17 +- .../yt_dlp/extractor/videomore.py | 13 +- .../yt_dlp/extractor/videopress.py | 6 +- .../yt_dlp/extractor/vidflex.py | 148 + .../yt_dlp/extractor/vidio.py | 24 +- .../yt_dlp/extractor/vidlii.py | 6 +- .../yt_dlp/extractor/vidly.py | 83 + .../yt_dlp/extractor/vidyard.py | 425 + .../yt_dlp/extractor/viewlift.py | 61 +- .../yt_dlp/extractor/viidea.py | 23 +- .../youtube_download/yt_dlp/extractor/viki.py | 346 - .../yt_dlp/extractor/vimeo.py | 581 +- .../yt_dlp/extractor/vimple.py | 58 - .../youtube_download/yt_dlp/extractor/vine.py | 151 - .../yt_dlp/extractor/viously.py | 60 + .../yt_dlp/extractor/viqeo.py | 3 +- .../youtube_download/yt_dlp/extractor/viu.py | 47 +- .../youtube_download/yt_dlp/extractor/vk.py | 210 +- .../yt_dlp/extractor/vocaroo.py | 2 +- .../yt_dlp/extractor/vodlocker.py | 73 - .../yt_dlp/extractor/vodplatform.py | 2 +- .../yt_dlp/extractor/voicerepublic.py | 59 - .../yt_dlp/extractor/voicy.py | 21 +- .../yt_dlp/extractor/volejtv.py | 4 +- .../youtube_download/yt_dlp/extractor/voot.py | 210 - .../yt_dlp/extractor/voxmedia.py | 17 +- .../youtube_download/yt_dlp/extractor/vrak.py | 77 - .../yt_dlp/extractor/vrsquare.py | 185 + .../youtube_download/yt_dlp/extractor/vrt.py | 525 +- .../youtube_download/yt_dlp/extractor/vrv.py | 269 - .../yt_dlp/extractor/vshare.py | 57 - .../youtube_download/yt_dlp/extractor/vtm.py | 5 +- .../youtube_download/yt_dlp/extractor/vtv.py | 108 + .../yt_dlp/extractor/vuclip.py | 10 +- .../yt_dlp/extractor/vupload.py | 52 - .../yt_dlp/extractor/vvvvid.py | 99 +- .../yt_dlp/extractor/vyborymos.py | 52 - .../yt_dlp/extractor/vzaar.py | 100 - .../yt_dlp/extractor/wakanim.py | 75 - .../yt_dlp/extractor/walla.py | 6 +- .../yt_dlp/extractor/wasdtv.py | 159 - .../yt_dlp/extractor/washingtonpost.py | 7 +- .../youtube_download/yt_dlp/extractor/wat.py | 17 +- .../yt_dlp/extractor/watchbox.py | 153 - .../yt_dlp/extractor/watchindianporn.py | 65 - .../youtube_download/yt_dlp/extractor/wdr.py | 51 +- .../yt_dlp/extractor/webcamerapl.py | 4 +- .../yt_dlp/extractor/webcaster.py | 2 +- .../yt_dlp/extractor/webofstories.py | 18 +- .../yt_dlp/extractor/weibo.py | 359 +- .../yt_dlp/extractor/weiqitv.py | 1 + .../yt_dlp/extractor/weverse.py | 97 +- .../yt_dlp/extractor/wevidi.py | 14 +- .../yt_dlp/extractor/whowatch.py | 23 +- .../yt_dlp/extractor/wikimedia.py | 4 +- .../yt_dlp/extractor/willow.py | 56 - .../yt_dlp/extractor/wimtv.py | 30 +- .../yt_dlp/extractor/wistia.py | 40 +- .../yt_dlp/extractor/wordpress.py | 16 +- .../yt_dlp/extractor/worldstarhiphop.py | 4 +- .../yt_dlp/extractor/wppilot.py | 16 +- .../yt_dlp/extractor/wrestleuniverse.py | 23 +- .../youtube_download/yt_dlp/extractor/wsj.py | 26 +- .../youtube_download/yt_dlp/extractor/wwe.py | 7 +- .../yt_dlp/extractor/wykop.py | 12 +- .../yt_dlp/extractor/xanimu.py | 19 +- .../youtube_download/yt_dlp/extractor/xbef.py | 42 - .../yt_dlp/extractor/xboxclips.py | 4 +- .../yt_dlp/extractor/xfileshare.py | 198 - .../yt_dlp/extractor/xhamster.py | 69 +- .../yt_dlp/extractor/xiaohongshu.py | 109 + .../yt_dlp/extractor/ximalaya.py | 123 +- .../yt_dlp/extractor/xinpianchang.py | 36 +- .../yt_dlp/extractor/xminus.py | 3 +- .../youtube_download/yt_dlp/extractor/xnxx.py | 4 +- .../yt_dlp/extractor/xstream.py | 9 +- .../yt_dlp/extractor/xtube.py | 214 - .../yt_dlp/extractor/xuite.py | 149 - .../yt_dlp/extractor/xvideos.py | 113 +- .../yt_dlp/extractor/xxxymovies.py | 4 +- .../yt_dlp/extractor/yahoo.py | 22 +- .../yt_dlp/extractor/yandexdisk.py | 8 +- .../yt_dlp/extractor/yandexmusic.py | 75 +- .../yt_dlp/extractor/yandexvideo.py | 174 +- .../yt_dlp/extractor/yapfiles.py | 7 +- .../yt_dlp/extractor/yappy.py | 13 +- .../yt_dlp/extractor/yesjapan.py | 56 - .../yt_dlp/extractor/yinyuetai.py | 52 - .../yt_dlp/extractor/yle_areena.py | 182 +- .../youtube_download/yt_dlp/extractor/ynet.py | 48 - .../yt_dlp/extractor/youjizz.py | 2 +- .../yt_dlp/extractor/youku.py | 23 +- .../yt_dlp/extractor/younow.py | 50 +- .../yt_dlp/extractor/youporn.py | 417 +- .../yt_dlp/extractor/yourporn.py | 65 - .../yt_dlp/extractor/yourupload.py | 43 - .../yt_dlp/extractor/youtube.py | 7332 ----------------- .../yt_dlp/extractor/youtube/__init__.py | 50 + .../yt_dlp/extractor/youtube/_base.py | 1102 +++ .../yt_dlp/extractor/youtube/_clip.py | 66 + .../yt_dlp/extractor/youtube/_mistakes.py | 69 + .../extractor/youtube/_notifications.py | 98 + .../yt_dlp/extractor/youtube/_redirect.py | 247 + .../yt_dlp/extractor/youtube/_search.py | 167 + .../yt_dlp/extractor/youtube/_tab.py | 2385 ++++++ .../yt_dlp/extractor/youtube/_video.py | 4106 +++++++++ .../yt_dlp/extractor/zaiko.py | 31 +- .../yt_dlp/extractor/zapiks.py | 8 +- .../yt_dlp/extractor/zattoo.py | 51 +- .../youtube_download/yt_dlp/extractor/zdf.py | 918 ++- .../youtube_download/yt_dlp/extractor/zee5.py | 41 +- .../yt_dlp/extractor/zeenews.py | 8 +- .../yt_dlp/extractor/zenporn.py | 118 + .../yt_dlp/extractor/zetland.py | 71 + .../yt_dlp/extractor/zhihu.py | 4 +- .../yt_dlp/extractor/zingmp3.py | 258 +- .../youtube_download/yt_dlp/extractor/zoom.py | 51 +- .../youtube_download/yt_dlp/extractor/zype.py | 8 +- plugins/youtube_download/yt_dlp/globals.py | 30 + plugins/youtube_download/yt_dlp/jsinterp.py | 128 +- .../yt_dlp/networking/__init__.py | 27 +- .../yt_dlp/networking/_curlcffi.py | 296 + .../yt_dlp/networking/_helper.py | 85 +- .../yt_dlp/networking/_requests.py | 426 + .../yt_dlp/networking/_urllib.py | 110 +- .../yt_dlp/networking/_websockets.py | 189 + .../yt_dlp/networking/common.py | 78 +- .../yt_dlp/networking/exceptions.py | 122 +- .../yt_dlp/networking/impersonate.py | 155 + .../yt_dlp/networking/websocket.py | 23 + plugins/youtube_download/yt_dlp/options.py | 214 +- plugins/youtube_download/yt_dlp/plugins.py | 183 +- .../yt_dlp/postprocessor/__init__.py | 35 +- .../yt_dlp/postprocessor/common.py | 12 +- .../yt_dlp/postprocessor/embedthumbnail.py | 54 +- .../yt_dlp/postprocessor/exec.py | 15 +- .../yt_dlp/postprocessor/ffmpeg.py | 99 +- .../yt_dlp/postprocessor/modify_chapters.py | 2 +- .../postprocessor/movefilesafterdownload.py | 23 +- .../yt_dlp/postprocessor/sponskrub.py | 11 +- .../yt_dlp/postprocessor/sponsorblock.py | 12 +- .../yt_dlp/postprocessor/xattrpp.py | 37 +- plugins/youtube_download/yt_dlp/socks.py | 41 +- plugins/youtube_download/yt_dlp/update.py | 536 +- .../yt_dlp/utils/_deprecated.py | 36 +- .../youtube_download/yt_dlp/utils/_legacy.py | 112 +- .../youtube_download/yt_dlp/utils/_utils.py | 854 +- .../yt_dlp/utils/networking.py | 153 +- .../youtube_download/yt_dlp/utils/progress.py | 109 + .../yt_dlp/utils/traversal.py | 279 +- plugins/youtube_download/yt_dlp/version.py | 8 +- plugins/youtube_download/yt_dlp/webvtt.py | 19 +- src/solarfm/__builtins__.py | 14 + src/solarfm/__main__.py | 9 +- src/solarfm/core/fs_actions/handler_mixin.py | 25 +- src/solarfm/core/mixins/ui/grid_mixin.py | 47 +- src/solarfm/core/ui_mixin.py | 3 +- src/solarfm/core/widgets/icon_grid_widget.py | 2 +- src/solarfm/core/window.py | 7 +- 1194 files changed, 60099 insertions(+), 44436 deletions(-) delete mode 100644 plugins/youtube_download/yt_dlp/casefold.py delete mode 100644 plugins/youtube_download/yt_dlp/compat/functools.py create mode 100644 plugins/youtube_download/yt_dlp/downloader/bunnycdn.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/airmozilla.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/allstar.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/altcensored.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/amadeustv.py rename plugins/youtube_download/yt_dlp/extractor/{ant1newsgr.py => antenna.py} (67%) create mode 100644 plugins/youtube_download/yt_dlp/extractor/art19.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/asiancrush.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/asobichannel.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/asobistage.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/atttechchannel.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/axs.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/bandlab.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/beacon.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/biqle.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/bitwave.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/bluesky.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/boosty.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/breakcom.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/brilliantpala.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/bundestag.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/bunnycdn.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/cableav.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/caffeinetv.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/camwithher.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/canal1.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/canalsurmas.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/caracoltv.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/carambatv.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/cbsinteractive.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/channel9.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/chingari.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/chirbit.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/chzzk.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/cinchcast.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/cineverse.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/cliphunter.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/clipsyndicate.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/cloudy.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/cloudycdn.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/crunchyroll.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/daftsex.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/dangalplay.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/deezer.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/defense.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/digg.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/digiview.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/discovery.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/discoverygo.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/dotsub.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/drtalks.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/duoplay.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/echomsk.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/eggs.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/ehow.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/einthusan.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/elementorembed.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/elevensports.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/ellentube.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/eltrecetv.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/engadget.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/epidemicsound.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/eplus.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/erocast.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/err.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/escapist.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/esri.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/expotv.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/extremetube.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/fathom.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/filmmodu.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/flextv.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/floatplane.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/fourzerostudio.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/foxgay.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/francaisfacile.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/funimation.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/fusion.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/gamedevtv.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/gameinformer.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/gbnews.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/germanupa.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/getcourseru.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/gfycat.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/giga.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/gigya.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/godresource.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/graspop.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/helsinki.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/hitbox.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/howcast.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/howstuffworks.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/ilpost.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/ivoox.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/jable.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/jiocinema.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/jiosaavn.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/joqrag.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/jtbc.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/kanal2.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/karrierevideos.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/keezmovies.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/kenh14.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/kika.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/konserthusetplay.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/kukululive.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/kusi.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/laola1tv.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/laracasts.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/laxarxames.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/learningonscreen.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/linuxacademy.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/lnk.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/lnkgo.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/localnews8.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/loco.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/loom.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/lsm.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/m6.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/maariv.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/magentamusik.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/magentamusik360.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/malltv.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/mbn.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/meta.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/metacafe.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/mgoon.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/miaopai.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/microsoftvirtualacademy.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/mildom.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/ministrygrid.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/miomio.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/mnet.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/moevideo.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/mofosex.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/mojevideo.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/monstercat.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/morningstar.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/movieclips.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/mwave.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/mx3.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/mychannels.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/myvi.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/nest.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/newstube.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/niconicochannelplus.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/ninaprotocol.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/ninenews.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/njpwworld.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/normalboots.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/nosvideo.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/nts.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/nuum.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/odatv.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/ooyala.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/pandoratv.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/parti.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/people.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/pialive.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/piramidetv.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/playfm.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/plays.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/playstuff.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/playvid.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/plvideo.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/pokemon.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/porn91.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/pornbox.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/porncom.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/pornez.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/pornhd.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/radiobremen.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/radiocomercial.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/radioradicale.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/rbmaradio.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/recurbate.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/redge.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/regiotv.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/rice.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/ridehome.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/rinsefm.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/roya.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/rtvnh.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/rudovideo.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/ruhd.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/savefrom.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/sbscokr.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/screenrec.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/seeker.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/sejmpl.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/sen.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/shared.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/sharepoint.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/snapchat.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/softwhiteunderbelly.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/spankwire.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/sproutvideo.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/streaks.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/streamcloud.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/streamff.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/subsplash.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/swrmediathek.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/taptap.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/tbsjp.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/tdslifeway.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/techtalks.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/theguardian.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/theta.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/thisav.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/tinypic.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/tokentube.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/trilulilu.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/trtworld.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/tunepk.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/turbo.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/tvnet.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/tvnow.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/tvw.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/twentyfourvideo.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/uliza.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/unscripted.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/veehd.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/veoh.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/vidbit.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/vidflex.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/vidly.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/vidyard.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/viki.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/vimple.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/vine.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/viously.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/vodlocker.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/voicerepublic.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/voot.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/vrak.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/vrsquare.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/vrv.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/vshare.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/vtv.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/vupload.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/vyborymos.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/vzaar.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/wakanim.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/wasdtv.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/watchbox.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/watchindianporn.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/willow.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/xbef.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/xfileshare.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/xiaohongshu.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/xtube.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/xuite.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/yesjapan.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/yinyuetai.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/ynet.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/yourporn.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/yourupload.py delete mode 100644 plugins/youtube_download/yt_dlp/extractor/youtube.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/youtube/__init__.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/youtube/_base.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/youtube/_clip.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/youtube/_mistakes.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/youtube/_notifications.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/youtube/_redirect.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/youtube/_search.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/youtube/_tab.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/youtube/_video.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/zenporn.py create mode 100644 plugins/youtube_download/yt_dlp/extractor/zetland.py create mode 100644 plugins/youtube_download/yt_dlp/globals.py create mode 100644 plugins/youtube_download/yt_dlp/networking/_curlcffi.py create mode 100644 plugins/youtube_download/yt_dlp/networking/_requests.py create mode 100644 plugins/youtube_download/yt_dlp/networking/_websockets.py create mode 100644 plugins/youtube_download/yt_dlp/networking/impersonate.py create mode 100644 plugins/youtube_download/yt_dlp/networking/websocket.py create mode 100644 plugins/youtube_download/yt_dlp/utils/progress.py diff --git a/plugins/youtube_download/download.sh b/plugins/youtube_download/download.sh index de8e1ea9..af038946 100755 --- a/plugins/youtube_download/download.sh +++ b/plugins/youtube_download/download.sh @@ -8,12 +8,29 @@ function main() { - cd "$(dirname "")" - echo "Working Dir: " $(pwd) + _STARGET="${1}" + _SPATH="${HOME}/.config/solarfm/plugins/youtube_download" LINK=`xclip -selection clipboard -o` - python "${HOME}/.config/solarfm/plugins/youtube_download/yt_dlp/__main__.py" \ - --write-sub --embed-sub --sub-langs en \ - -o "${1}/%(title)s.%(ext)s" "${LINK}" + cd "${_SPATH}" + echo "Working Dir: " $(pwd) + + rm "${_SPATH}/../../cookies.txt" + + # Note: Export cookies to file + python "${_SPATH}/yt_dlp/__main__.py" \ + --cookies-from-browser firefox --cookies "${_SPATH}/../../cookies.txt" + + # Note: Use cookies from browser directly + # python "${_SPATH}/yt_dlp/__main__.py" \ + # --cookies-from-browser firefox --write-sub --embed-sub --sub-langs en \ + # -o "${_STARGET}/%(title)s.%(ext)s" "${LINK}" + + # Note: Download video + python "${_SPATH}/yt_dlp/__main__.py" \ + -f "bestvideo[height<=1080][ext=mp4][vcodec^=avc]+bestaudio[ext=m4a]/best[ext=mp4]/best" \ + --cookies "${_SPATH}/../../cookies.txt" --write-sub --embed-sub --sub-langs en \ + -o "${_STARGET}/%(title)s.%(ext)s" "${LINK}" + } -main "$@"; +main "$@"; \ No newline at end of file diff --git a/plugins/youtube_download/yt_dlp/YoutubeDL.py b/plugins/youtube_download/yt_dlp/YoutubeDL.py index 666d89b4..63e6e11b 100644 --- a/plugins/youtube_download/yt_dlp/YoutubeDL.py +++ b/plugins/youtube_download/yt_dlp/YoutubeDL.py @@ -1,9 +1,10 @@ import collections import contextlib import copy -import datetime +import datetime as dt import errno import fileinput +import functools import http.cookiejar import io import itertools @@ -24,14 +25,23 @@ import traceback import unicodedata from .cache import Cache -from .compat import functools, urllib # isort: split -from .compat import compat_os_name, compat_shlex_quote, urllib_req_to_req -from .cookies import LenientSimpleCookie, load_cookies +from .compat import urllib # isort: split +from .compat import urllib_req_to_req +from .cookies import CookieLoadError, LenientSimpleCookie, load_cookies from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name from .downloader.rtmp import rtmpdump_version -from .extractor import gen_extractor_classes, get_info_extractor +from .extractor import gen_extractor_classes, get_info_extractor, import_extractors from .extractor.common import UnsupportedURLIE from .extractor.openload import PhantomJSwrapper +from .globals import ( + IN_CLI, + LAZY_EXTRACTORS, + plugin_ies, + plugin_ies_overrides, + plugin_pps, + all_plugins_loaded, + plugin_dirs, +) from .minicurses import format_text from .networking import HEADRequest, Request, RequestDirector from .networking.common import _REQUEST_HANDLERS, _RH_PREFERENCES @@ -40,11 +50,10 @@ from .networking.exceptions import ( NoSupportingHandlers, RequestError, SSLError, - _CompatHTTPError, network_exceptions, ) -from .plugins import directories as plugin_directories -from .postprocessor import _PLUGIN_CLASSES as plugin_pps +from .networking.impersonate import ImpersonateRequestHandler +from .plugins import directories as plugin_directories, load_all_plugins from .postprocessor import ( EmbedThumbnailPP, FFmpegFixupDuplicateMoovPP, @@ -60,7 +69,13 @@ from .postprocessor import ( get_postprocessor, ) from .postprocessor.ffmpeg import resolve_mapping as resolve_recode_mapping -from .update import REPOSITORY, current_git_head, detect_variant +from .update import ( + REPOSITORY, + _get_system_deprecation, + _make_label, + current_git_head, + detect_variant, +) from .utils import ( DEFAULT_OUTTMPL, IDENTITY, @@ -94,16 +109,14 @@ from .utils import ( SameFileError, UnavailableVideoError, UserNotLive, + YoutubeDLError, age_restricted, - args_to_str, bug_reports_message, date_from_str, deprecation_warning, determine_ext, determine_protocol, encode_compat_str, - encodeFilename, - error_to_compat_str, escapeHTML, expand_path, extract_basic_auth, @@ -134,11 +147,13 @@ from .utils import ( sanitize_filename, sanitize_path, sanitize_url, + shell_quote, str_or_none, strftime_or_none, subtitles_filename, supports_terminal_sequences, system_identifier, + filesize_from_tbr, timetuple_from_msec, to_high_limit_path, traverse_obj, @@ -146,24 +161,37 @@ from .utils import ( try_get, url_basename, variadic, - version_tuple, windows_enable_vt_mode, write_json_file, write_string, ) -from .utils._utils import _YDLLogger +from .utils._utils import _UnsafeExtensionError, _YDLLogger, _ProgressState from .utils.networking import ( HTTPHeaderDict, clean_headers, clean_proxies, std_headers, ) -from .version import CHANNEL, RELEASE_GIT_HEAD, VARIANT, __version__ +from .version import CHANNEL, ORIGIN, RELEASE_GIT_HEAD, VARIANT, __version__ -if compat_os_name == 'nt': +if os.name == 'nt': import ctypes +def _catch_unsafe_extension_error(func): + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + try: + return func(self, *args, **kwargs) + except _UnsafeExtensionError as error: + self.report_error( + f'The extracted extension ({error.extension!r}) is unusual ' + 'and will be skipped for safety reasons. ' + f'If you believe this is an error{bug_reports_message(",")}') + + return wrapper + + class YoutubeDL: """YoutubeDL class. @@ -229,7 +257,7 @@ class YoutubeDL: format_sort_force: Force the given format_sort. see "Sorting Formats" for more details. prefer_free_formats: Whether to prefer video formats with free containers - over non-free ones of same quality. + over non-free ones of the same quality. allow_multiple_video_streams: Allow multiple video streams to be merged into a single file allow_multiple_audio_streams: Allow multiple audio streams to be merged @@ -239,14 +267,16 @@ class YoutubeDL: 'selected' (check selected formats), or None (check only if requested by extractor) paths: Dictionary of output paths. The allowed keys are 'home' - 'temp' and the keys of OUTTMPL_TYPES (in utils.py) + 'temp' and the keys of OUTTMPL_TYPES (in utils/_utils.py) outtmpl: Dictionary of templates for output names. Allowed keys - are 'default' and the keys of OUTTMPL_TYPES (in utils.py). + are 'default' and the keys of OUTTMPL_TYPES (in utils/_utils.py). For compatibility with youtube-dl, a single string can also be used outtmpl_na_placeholder: Placeholder for unavailable meta fields. restrictfilenames: Do not allow "&" and spaces in file names trim_file_name: Limit length of filename (extension excluded) - windowsfilenames: Force the filenames to be windows compatible + windowsfilenames: True: Force filenames to be Windows compatible + False: Sanitize filenames only minimally + This option has no effect when running on Windows ignoreerrors: Do not stop on download/postprocessing errors. Can be 'only_download' to ignore only download errors. Default is 'only_download' for CLI, but False for API @@ -261,9 +291,12 @@ class YoutubeDL: lazy_playlist: Process playlist entries as they are received. matchtitle: Download only matching titles. rejecttitle: Reject downloads for matching titles. - logger: Log messages to a logging.Logger instance. + logger: A class having a `debug`, `warning` and `error` function where + each has a single string parameter, the message to be logged. + For compatibility reasons, both debug and info messages are passed to `debug`. + A debug message will have a prefix of `[debug] ` to discern it from info messages. logtostderr: Print everything to stderr instead of stdout. - consoletitle: Display progress in console window's titlebar. + consoletitle: Display progress in the console window's titlebar. writedescription: Write the video description to a .description file writeinfojson: Write the video description to a .info.json file clean_infojson: Remove internal metadata from the infojson @@ -397,6 +430,8 @@ class YoutubeDL: - "detect_or_warn": check whether we can do anything about it, warn otherwise (default) source_address: Client-side IP address to bind to. + impersonate: Client to impersonate for requests. + An ImpersonateTarget (from yt_dlp.networking.impersonate) sleep_interval_requests: Number of seconds to sleep between requests during extraction sleep_interval: Number of seconds to sleep before each download when @@ -422,13 +457,14 @@ class YoutubeDL: asked whether to download the video. - Raise utils.DownloadCancelled(msg) to abort remaining downloads when a video is rejected. - match_filter_func in utils.py is one example for this. + match_filter_func in utils/_utils.py is one example for this. color: A Dictionary with output stream names as keys and their respective color policy as values. Can also just be a single color policy, in which case it applies to all outputs. Valid stream names are 'stdout' and 'stderr'. - Valid color policies are one of 'always', 'auto', 'no_color' or 'never'. + Valid color policies are one of 'always', 'auto', + 'no_color', 'never', 'auto-tty' or 'no_color-tty'. geo_bypass: Bypass geographic restriction via faking X-Forwarded-For HTTP header geo_bypass_country: @@ -444,8 +480,9 @@ class YoutubeDL: Set the value to 'native' to use the native downloader compat_opts: Compatibility options. See "Differences in default behavior". The following options do not work when used through the API: - filename, abort-on-error, multistreams, no-live-chat, format-sort - no-clean-infojson, no-playlist-metafiles, no-keep-subs, no-attach-info-json. + filename, abort-on-error, multistreams, no-live-chat, + format-sort, no-clean-infojson, no-playlist-metafiles, + no-keep-subs, no-attach-info-json, allow-unsafe-ext, prefer-vp9-sort. Refer __init__.py for their implementation progress_template: Dictionary of templates for progress outputs. Allowed keys are 'download', 'postprocess', @@ -471,7 +508,7 @@ class YoutubeDL: nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize, max_filesize, test, noresizebuffer, retries, file_access_retries, fragment_retries, continuedl, xattr_set_filesize, hls_use_mpegts, http_chunk_size, - external_downloader_args, concurrent_fragment_downloads. + external_downloader_args, concurrent_fragment_downloads, progress_delta. The following options are used by the post processors: ffmpeg_location: Location of the ffmpeg/avconv binary; either the path @@ -487,7 +524,7 @@ class YoutubeDL: The following options are used by the extractors: extractor_retries: Number of times to retry for known errors (default: 3) dynamic_mpd: Whether to process dynamic DASH manifests (default: True) - hls_split_discontinuity: Split HLS playlists to different formats at + hls_split_discontinuity: Split HLS playlists into different formats at discontinuities such as ad breaks (default: False) extractor_args: A dictionary of arguments to be passed to the extractors. See "EXTRACTOR ARGUMENTS" for details. @@ -527,7 +564,7 @@ class YoutubeDL: include_ads: - Doesn't work Download ads as well call_home: - Not implemented - Boolean, true iff we are allowed to contact the + Boolean, true if we are allowed to contact the yt-dlp servers for debugging. post_hooks: - Register a custom postprocessor A list of functions that get called as the final step @@ -569,15 +606,23 @@ class YoutubeDL: # NB: Keep in sync with the docstring of extractor/common.py 'url', 'manifest_url', 'manifest_stream_number', 'ext', 'format', 'format_id', 'format_note', 'width', 'height', 'aspect_ratio', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'audio_channels', - 'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns', - 'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start', + 'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns', 'hls_media_playlist_data', + 'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start', 'is_dash_periods', 'request_data', 'preference', 'language', 'language_preference', 'quality', 'source_preference', 'cookies', - 'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'extra_param_to_segment_url', 'hls_aes', 'downloader_options', - 'page_url', 'app', 'play_path', 'tc_url', 'flash_version', 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time' + 'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'extra_param_to_segment_url', 'extra_param_to_key_url', + 'hls_aes', 'downloader_options', 'page_url', 'app', 'play_path', 'tc_url', 'flash_version', + 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time', + } + _deprecated_multivalue_fields = { + 'album_artist': 'album_artists', + 'artist': 'artists', + 'composer': 'composers', + 'creator': 'creators', + 'genre': 'genres', } _format_selection_exts = { 'audio': set(MEDIA_EXTENSIONS.common_audio), - 'video': set(MEDIA_EXTENSIONS.common_video + ('3gp', )), + 'video': {*MEDIA_EXTENSIONS.common_video, '3gp'}, 'storyboards': set(MEDIA_EXTENSIONS.storyboards), } @@ -605,13 +650,15 @@ class YoutubeDL: self.cache = Cache(self) self.__header_cookies = [] + # compat for API: load plugins if they have not already + if not all_plugins_loaded.value: + load_all_plugins() + stdout = sys.stderr if self.params.get('logtostderr') else sys.stdout self._out_files = Namespace( out=stdout, error=sys.stderr, screen=sys.stderr if self.params.get('quiet') else stdout, - console=None if compat_os_name == 'nt' else next( - filter(supports_terminal_sequences, (sys.stderr, sys.stdout)), None) ) try: @@ -619,19 +666,28 @@ class YoutubeDL: except Exception as e: self.write_debug(f'Failed to enable VT mode: {e}') + # hehe "immutable" namespace + self._out_files.console = next(filter(supports_terminal_sequences, (sys.stderr, sys.stdout)), None) + if self.params.get('no_color'): if self.params.get('color') is not None: self.params.setdefault('_warnings', []).append( 'Overwriting params from "color" with "no_color"') self.params['color'] = 'no_color' - term_allow_color = os.environ.get('TERM', '').lower() != 'dumb' + term_allow_color = os.getenv('TERM', '').lower() != 'dumb' + base_no_color = bool(os.getenv('NO_COLOR')) def process_color_policy(stream): stream_name = {sys.stdout: 'stdout', sys.stderr: 'stderr'}[stream] - policy = traverse_obj(self.params, ('color', (stream_name, None), {str}), get_all=False) - if policy in ('auto', None): - return term_allow_color and supports_terminal_sequences(stream) + policy = traverse_obj(self.params, ('color', (stream_name, None), {str}, any)) or 'auto' + if policy in ('auto', 'auto-tty', 'no_color-tty'): + no_color = base_no_color + if policy.endswith('tty'): + no_color = policy.startswith('no_color') + if term_allow_color and supports_terminal_sequences(stream): + return 'no_color' if no_color else True + return False assert policy in ('always', 'never', 'no_color'), policy return {'always': True, 'never': False}.get(policy, policy) @@ -640,17 +696,9 @@ class YoutubeDL: for name, stream in self._out_files.items_ if name != 'console' }) - # The code is left like this to be reused for future deprecations - MIN_SUPPORTED, MIN_RECOMMENDED = (3, 7), (3, 7) - current_version = sys.version_info[:2] - if current_version < MIN_RECOMMENDED: - msg = ('Support for Python version %d.%d has been deprecated. ' - 'See https://github.com/yt-dlp/yt-dlp/issues/3764 for more details.' - '\n You will no longer receive updates on this version') - if current_version < MIN_SUPPORTED: - msg = 'Python version %d.%d is no longer supported' - self.deprecated_feature( - f'{msg}! Please update to Python %d.%d or above' % (*current_version, *MIN_RECOMMENDED)) + system_deprecation = _get_system_deprecation() + if system_deprecation: + self.deprecated_feature(system_deprecation.replace('\n', '\n ')) if self.params.get('allow_unplayable_formats'): self.report_warning( @@ -667,9 +715,9 @@ class YoutubeDL: width_args = [] if width is None else ['-w', str(width)] sp_kwargs = {'stdin': subprocess.PIPE, 'stdout': slave, 'stderr': self._out_files.error} try: - self._output_process = Popen(['bidiv'] + width_args, **sp_kwargs) + self._output_process = Popen(['bidiv', *width_args], **sp_kwargs) except OSError: - self._output_process = Popen(['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs) + self._output_process = Popen(['fribidi', '-c', 'UTF-8', *width_args], **sp_kwargs) self._output_channel = os.fdopen(master, 'rb') except OSError as ose: if ose.errno == errno.ENOENT: @@ -683,7 +731,6 @@ class YoutubeDL: self.params['http_headers'] = HTTPHeaderDict(std_headers, self.params.get('http_headers')) self._load_cookies(self.params['http_headers'].get('Cookie')) # compat self.params['http_headers'].pop('Cookie', None) - self._request_director = self.build_request_director(_REQUEST_HANDLERS.values(), _RH_PREFERENCES) if auto_init and auto_init != 'no_verbose_header': self.print_debug_header() @@ -707,6 +754,13 @@ class YoutubeDL: for msg in self.params.get('_deprecation_warnings', []): self.deprecated_feature(msg) + if impersonate_target := self.params.get('impersonate'): + if not self._impersonate_target_available(impersonate_target): + raise YoutubeDLError( + f'Impersonate target "{impersonate_target}" is not available. ' + f'Use --list-impersonate-targets to see available targets. ' + f'You may be missing dependencies required to support this target.') + if 'list-formats' in self.params['compat_opts']: self.params['listformats_table'] = False @@ -804,8 +858,7 @@ class YoutubeDL: ) self.report_warning( 'Long argument string detected. ' - 'Use -- to separate parameters and URLs, like this:\n%s' % - args_to_str(correct_argv)) + f'Use -- to separate parameters and URLs, like this:\n{shell_quote(correct_argv)}') def add_info_extractor(self, ie): """Add an InfoExtractor object to the end of the list.""" @@ -904,7 +957,7 @@ class YoutubeDL: if (self.params.get('quiet') if quiet is None else quiet) and not self.params.get('verbose'): return self._write_string( - '%s%s' % (self._bidi_workaround(message), ('' if skip_eol else '\n')), + '{}{}'.format(self._bidi_workaround(message), ('' if skip_eol else '\n')), self._out_files.screen, only_once=only_once) def to_stderr(self, message, only_once=False): @@ -916,21 +969,22 @@ class YoutubeDL: self._write_string(f'{self._bidi_workaround(message)}\n', self._out_files.error, only_once=only_once) def _send_console_code(self, code): - if compat_os_name == 'nt' or not self._out_files.console: - return + if not supports_terminal_sequences(self._out_files.console): + return False self._write_string(code, self._out_files.console) + return True - def to_console_title(self, message): - if not self.params.get('consoletitle', False): + def to_console_title(self, message=None, progress_state=None, percent=None): + if not self.params.get('consoletitle'): return - message = remove_terminal_sequences(message) - if compat_os_name == 'nt': - if ctypes.windll.kernel32.GetConsoleWindow(): - # c_wchar_p() might not be necessary if `message` is - # already of type unicode() - ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message)) - else: - self._send_console_code(f'\033]0;{message}\007') + + if message: + success = self._send_console_code(f'\033]0;{remove_terminal_sequences(message)}\007') + if not success and os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow(): + ctypes.windll.kernel32.SetConsoleTitleW(message) + + if isinstance(progress_state, _ProgressState): + self._send_console_code(progress_state.get_ansi_escape(percent)) def save_console_title(self): if not self.params.get('consoletitle') or self.params.get('simulate'): @@ -944,6 +998,7 @@ class YoutubeDL: def __enter__(self): self.save_console_title() + self.to_console_title(progress_state=_ProgressState.INDETERMINATE) return self def save_cookies(self): @@ -952,11 +1007,14 @@ class YoutubeDL: def __exit__(self, *args): self.restore_console_title() + self.to_console_title(progress_state=_ProgressState.HIDDEN) self.close() def close(self): self.save_cookies() - self._request_director.close() + if '_request_director' in self.__dict__: + self._request_director.close() + del self._request_director def trouble(self, message=None, tb=None, is_error=True): """Determine action to take when a download problem appears. @@ -1025,10 +1083,10 @@ class YoutubeDL: return self._format_text(self._out_files.error, self._allow_colors.error, *args, **kwargs) def report_warning(self, message, only_once=False): - ''' + """ Print the message to stderr, it will be prefixed with 'WARNING:' If stderr is a tty file the 'WARNING:' will be colored - ''' + """ if self.params.get('logger') is not None: self.params['logger'].warning(message) else: @@ -1046,14 +1104,14 @@ class YoutubeDL: self.to_stderr(f'{self._format_err("Deprecated Feature:", self.Styles.ERROR)} {message}', True) def report_error(self, message, *args, **kwargs): - ''' + """ Do the same as trouble, but prefixes the message with 'ERROR:', colored in red if stderr is a tty file. - ''' + """ self.trouble(f'{self._format_err("ERROR:", self.Styles.ERROR)} {message}', *args, **kwargs) def write_debug(self, message, only_once=False): - '''Log debug message or Print message to stderr''' + """Log debug message or Print message to stderr""" if not self.params.get('verbose', False): return message = f'[debug] {message}' @@ -1065,21 +1123,21 @@ class YoutubeDL: def report_file_already_downloaded(self, file_name): """Report file has already been fully downloaded.""" try: - self.to_screen('[download] %s has already been downloaded' % file_name) + self.to_screen(f'[download] {file_name} has already been downloaded') except UnicodeEncodeError: self.to_screen('[download] The file has already been downloaded') def report_file_delete(self, file_name): """Report that existing file will be deleted.""" try: - self.to_screen('Deleting existing file %s' % file_name) + self.to_screen(f'Deleting existing file {file_name}') except UnicodeEncodeError: self.to_screen('Deleting existing file') def raise_no_formats(self, info, forced=False, *, msg=None): has_drm = info.get('_has_drm') ignored, expected = self.params.get('ignore_no_formats_error'), bool(msg) - msg = msg or has_drm and 'This video is DRM protected' or 'No video formats found!' + msg = msg or (has_drm and 'This video is DRM protected') or 'No video formats found!' if forced or not ignored: raise ExtractorError(msg, video_id=info['id'], ie=info['extractor'], expected=has_drm or ignored or expected) @@ -1127,7 +1185,7 @@ class YoutubeDL: @staticmethod def escape_outtmpl(outtmpl): - ''' Escape any remaining strings like %s, %abc% etc. ''' + """ Escape any remaining strings like %s, %abc% etc. """ return re.sub( STR_FORMAT_RE_TMPL.format('', '(?![%(\0])'), lambda mobj: ('' if mobj.group('has_key') else '%') + mobj.group(0), @@ -1135,7 +1193,7 @@ class YoutubeDL: @classmethod def validate_outtmpl(cls, outtmpl): - ''' @return None or Exception object ''' + """ @return None or Exception object """ outtmpl = re.sub( STR_FORMAT_RE_TMPL.format('[^)]*', '[ljhqBUDS]'), lambda mobj: f'{mobj.group(0)[:-1]}s', @@ -1155,8 +1213,7 @@ class YoutubeDL: def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False): """ Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict - @param sanitize Whether to sanitize the output as a filename. - For backward compatibility, a function can also be passed + @param sanitize Whether to sanitize the output as a filename """ info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set @@ -1184,16 +1241,17 @@ class YoutubeDL: MATH_FUNCTIONS = { '+': float.__add__, '-': float.__sub__, + '*': float.__mul__, } # Field is of the form key1.key2... # where keys (except first) can be string, int, slice or "{field, ...}" - FIELD_INNER_RE = r'(?:\w+|%(num)s|%(num)s?(?::%(num)s?){1,2})' % {'num': r'(?:-?\d+)'} - FIELD_RE = r'\w*(?:\.(?:%(inner)s|{%(field)s(?:,%(field)s)*}))*' % { + FIELD_INNER_RE = r'(?:\w+|%(num)s|%(num)s?(?::%(num)s?){1,2})' % {'num': r'(?:-?\d+)'} # noqa: UP031 + FIELD_RE = r'\w*(?:\.(?:%(inner)s|{%(field)s(?:,%(field)s)*}))*' % { # noqa: UP031 'inner': FIELD_INNER_RE, - 'field': rf'\w*(?:\.{FIELD_INNER_RE})*' + 'field': rf'\w*(?:\.{FIELD_INNER_RE})*', } MATH_FIELD_RE = rf'(?:{FIELD_RE}|-?{NUMBER_RE})' - MATH_OPERATORS_RE = r'(?:%s)' % '|'.join(map(re.escape, MATH_FUNCTIONS.keys())) + MATH_OPERATORS_RE = r'(?:{})'.format('|'.join(map(re.escape, MATH_FUNCTIONS.keys()))) INTERNAL_FORMAT_RE = re.compile(rf'''(?xs) (?P-)? (?P{FIELD_RE}) @@ -1205,6 +1263,15 @@ class YoutubeDL: (?:\|(?P.*?))? )$''') + def _from_user_input(field): + if field == ':': + return ... + elif ':' in field: + return slice(*map(int_or_none, field.split(':'))) + elif int_or_none(field) is not None: + return int(field) + return field + def _traverse_infodict(fields): fields = [f for x in re.split(r'\.({.+?})\.?', fields) for f in ([x] if x.startswith('{') else x.split('.'))] @@ -1214,11 +1281,12 @@ class YoutubeDL: for i, f in enumerate(fields): if not f.startswith('{'): + fields[i] = _from_user_input(f) continue assert f.endswith('}'), f'No closing brace for {f} in {fields}' - fields[i] = {k: k.split('.') for k in f[1:-1].split(',')} + fields[i] = {k: list(map(_from_user_input, k.split('.'))) for k in f[1:-1].split(',')} - return traverse_obj(info_dict, fields, is_user_input=True, traverse_string=True) + return traverse_obj(info_dict, fields, traverse_string=True) def get_value(mdict): # Object traversal @@ -1261,14 +1329,23 @@ class YoutubeDL: na = self.params.get('outtmpl_na_placeholder', 'NA') - def filename_sanitizer(key, value, restricted=self.params.get('restrictfilenames')): + def filename_sanitizer(key, value, restricted): return sanitize_filename(str(value), restricted=restricted, is_id=( bool(re.search(r'(^|[_.])id(\.|$)', key)) if 'filename-sanitization' in self.params['compat_opts'] else NO_DEFAULT)) - sanitizer = sanitize if callable(sanitize) else filename_sanitizer - sanitize = bool(sanitize) + if callable(sanitize): + self.deprecation_warning('Passing a callable "sanitize" to YoutubeDL.prepare_outtmpl is deprecated') + elif not sanitize: + pass + elif (sys.platform != 'win32' and not self.params.get('restrictfilenames') + and self.params.get('windowsfilenames') is False): + def sanitize(key, value): + return str(value).replace('/', '\u29F8').replace('\0', '') + else: + def sanitize(key, value): + return filename_sanitizer(key, value, restricted=self.params.get('restrictfilenames')) def _dumpjson_default(obj): if isinstance(obj, (set, LazyList)): @@ -1306,7 +1383,7 @@ class YoutubeDL: value, default = None, na fmt = outer_mobj.group('format') - if fmt == 's' and last_field in field_size_compat_map.keys() and isinstance(value, int): + if fmt == 's' and last_field in field_size_compat_map and isinstance(value, int): fmt = f'0{field_size_compat_map[last_field]:d}d' flags = outer_mobj.group('conversion') or '' @@ -1324,14 +1401,14 @@ class YoutubeDL: value, fmt = escapeHTML(str(value)), str_fmt elif fmt[-1] == 'q': # quoted value = map(str, variadic(value) if '#' in flags else [value]) - value, fmt = ' '.join(map(compat_shlex_quote, value)), str_fmt + value, fmt = shell_quote(value, shell=True), str_fmt elif fmt[-1] == 'B': # bytes value = f'%{str_fmt}'.encode() % str(value).encode() value, fmt = value.decode('utf-8', 'ignore'), 's' elif fmt[-1] == 'U': # unicode normalized value, fmt = unicodedata.normalize( # "+" = compatibility equivalence, "#" = NFD - 'NF%s%s' % ('K' if '+' in flags else '', 'D' if '#' in flags else 'C'), + 'NF{}{}'.format('K' if '+' in flags else '', 'D' if '#' in flags else 'C'), value), str_fmt elif fmt[-1] == 'D': # decimal suffix num_fmt, fmt = fmt[:-1].replace('#', ''), 's' @@ -1351,15 +1428,15 @@ class YoutubeDL: if sanitize: # If value is an object, sanitize might convert it to a string - # So we convert it to repr first + # So we manually convert it before sanitizing if fmt[-1] == 'r': value, fmt = repr(value), str_fmt elif fmt[-1] == 'a': value, fmt = ascii(value), str_fmt if fmt[-1] in 'csra': - value = sanitizer(last_field, value) + value = sanitize(last_field, value) - key = '%s\0%s' % (key.replace('%', '%\0'), outer_mobj.group('format')) + key = '{}\0{}'.format(key.replace('%', '%\0'), outer_mobj.group('format')) TMPL_DICT[key] = value return '{prefix}%({key}){fmt}'.format(key=key, fmt=fmt, prefix=outer_mobj.group('prefix')) @@ -1369,6 +1446,7 @@ class YoutubeDL: outtmpl, info_dict = self.prepare_outtmpl(outtmpl, info_dict, *args, **kwargs) return self.escape_outtmpl(outtmpl) % info_dict + @_catch_unsafe_extension_error def _prepare_filename(self, info_dict, *, outtmpl=None, tmpl_type=None): assert None in (outtmpl, tmpl_type), 'outtmpl and tmpl_type are mutually exclusive' if outtmpl is None: @@ -1448,9 +1526,9 @@ class YoutubeDL: date = info_dict.get('upload_date') if date is not None: - dateRange = self.params.get('daterange', DateRange()) - if date not in dateRange: - return f'{date_from_str(date).isoformat()} upload date is not in range {dateRange}' + date_range = self.params.get('daterange', DateRange()) + if date not in date_range: + return f'{date_from_str(date).isoformat()} upload date is not in range {date_range}' view_count = info_dict.get('view_count') if view_count is not None: min_views = self.params.get('min_views') @@ -1460,7 +1538,7 @@ class YoutubeDL: if max_views is not None and view_count > max_views: return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views) if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')): - return 'Skipping "%s" because it is age restricted' % video_title + return f'Skipping "{video_title}" because it is age restricted' match_filter = self.params.get('match_filter') if match_filter is None: @@ -1513,7 +1591,7 @@ class YoutubeDL: @staticmethod def add_extra_info(info_dict, extra_info): - '''Set the keys from extra_info in info dict if they are missing''' + """Set the keys from extra_info in info dict if they are missing""" for key, value in extra_info.items(): info_dict.setdefault(key, value) @@ -1559,7 +1637,7 @@ class YoutubeDL: self.to_screen(f'[download] {self._format_screen(temp_id, self.Styles.ID)}: ' 'has already been recorded in the archive') if self.params.get('break_on_existing', False): - raise ExistingVideoReached() + raise ExistingVideoReached break return self.__extract_info(url, self.get_info_extractor(key), download, extra_info, process) else: @@ -1573,7 +1651,7 @@ class YoutubeDL: while True: try: return func(self, *args, **kwargs) - except (DownloadCancelled, LazyList.IndexError, PagedList.IndexError): + except (CookieLoadError, DownloadCancelled, LazyList.IndexError, PagedList.IndexError): raise except ReExtractInfo as e: if e.expected: @@ -1585,8 +1663,8 @@ class YoutubeDL: except GeoRestrictedError as e: msg = e.msg if e.countries: - msg += '\nThis video is available in %s.' % ', '.join( - map(ISO3166Utils.short2full, e.countries)) + msg += '\nThis video is available in {}.'.format(', '.join( + map(ISO3166Utils.short2full, e.countries))) msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.' self.report_error(msg) except ExtractorError as e: # An error we somewhat expected @@ -1795,8 +1873,8 @@ class YoutubeDL: if isinstance(additional_urls, str): additional_urls = [additional_urls] self.to_screen( - '[info] %s: %d additional URL(s) requested' % (ie_result['id'], len(additional_urls))) - self.write_debug('Additional URLs: "%s"' % '", "'.join(additional_urls)) + '[info] {}: {} additional URL(s) requested'.format(ie_result['id'], len(additional_urls))) + self.write_debug('Additional URLs: "{}"'.format('", "'.join(additional_urls))) ie_result['additional_entries'] = [ self.extract_info( url, download, extra_info=extra_info, @@ -1848,8 +1926,8 @@ class YoutubeDL: webpage_url = ie_result.get('webpage_url') # Playlists maynot have webpage_url if webpage_url and webpage_url in self._playlist_urls: self.to_screen( - '[download] Skipping already downloaded playlist: %s' - % ie_result.get('title') or ie_result.get('id')) + '[download] Skipping already downloaded playlist: {}'.format( + ie_result.get('title')) or ie_result.get('id')) return self._playlist_level += 1 @@ -1864,8 +1942,8 @@ class YoutubeDL: self._playlist_urls.clear() elif result_type == 'compat_list': self.report_warning( - 'Extractor %s returned a compat_list result. ' - 'It needs to be updated.' % ie_result.get('extractor')) + 'Extractor {} returned a compat_list result. ' + 'It needs to be updated.'.format(ie_result.get('extractor'))) def _fixup(r): self.add_extra_info(r, { @@ -1882,7 +1960,7 @@ class YoutubeDL: ] return ie_result else: - raise Exception('Invalid result type: %s' % result_type) + raise Exception(f'Invalid result type: {result_type}') def _ensure_dir_exists(self, path): return make_dir(path, self.report_error) @@ -1896,6 +1974,9 @@ class YoutubeDL: 'playlist_title': ie_result.get('title'), 'playlist_uploader': ie_result.get('uploader'), 'playlist_uploader_id': ie_result.get('uploader_id'), + 'playlist_channel': ie_result.get('channel'), + 'playlist_channel_id': ie_result.get('channel_id'), + 'playlist_webpage_url': ie_result.get('webpage_url'), **kwargs, } if strict: @@ -1998,8 +2079,9 @@ class YoutubeDL: resolved_entries[i] = (playlist_index, NO_DEFAULT) continue - self.to_screen('[download] Downloading item %s of %s' % ( - self._format_screen(i + 1, self.Styles.ID), self._format_screen(n_entries, self.Styles.EMPHASIS))) + self.to_screen( + f'[download] Downloading item {self._format_screen(i + 1, self.Styles.ID)} ' + f'of {self._format_screen(n_entries, self.Styles.EMPHASIS)}') entry_result = self.__process_iterable_entry(entry, download, collections.ChainMap({ 'playlist_index': playlist_index, @@ -2049,20 +2131,20 @@ class YoutubeDL: } operator_rex = re.compile(r'''(?x)\s* (?P[\w.-]+)\s* - (?P%s)(?P\s*\?)?\s* + (?P{})(?P\s*\?)?\s* (?P[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)\s* - ''' % '|'.join(map(re.escape, OPERATORS.keys()))) + '''.format('|'.join(map(re.escape, OPERATORS.keys())))) m = operator_rex.fullmatch(filter_spec) if m: try: - comparison_value = int(m.group('value')) + comparison_value = float(m.group('value')) except ValueError: comparison_value = parse_filesize(m.group('value')) if comparison_value is None: comparison_value = parse_filesize(m.group('value') + 'B') if comparison_value is None: raise ValueError( - 'Invalid value %r in format specification %r' % ( + 'Invalid value {!r} in format specification {!r}'.format( m.group('value'), filter_spec)) op = OPERATORS[m.group('op')] @@ -2072,15 +2154,15 @@ class YoutubeDL: '^=': lambda attr, value: attr.startswith(value), '$=': lambda attr, value: attr.endswith(value), '*=': lambda attr, value: value in attr, - '~=': lambda attr, value: value.search(attr) is not None + '~=': lambda attr, value: value.search(attr) is not None, } str_operator_rex = re.compile(r'''(?x)\s* (?P[a-zA-Z0-9._-]+)\s* - (?P!\s*)?(?P%s)\s*(?P\?\s*)? + (?P!\s*)?(?P{})\s*(?P\?\s*)? (?P["'])? (?P(?(quote)(?:(?!(?P=quote))[^\\]|\\.)+|[\w.-]+)) (?(quote)(?P=quote))\s* - ''' % '|'.join(map(re.escape, STR_OPERATORS.keys()))) + '''.format('|'.join(map(re.escape, STR_OPERATORS.keys())))) m = str_operator_rex.fullmatch(filter_spec) if m: if m.group('op') == '~=': @@ -2094,7 +2176,7 @@ class YoutubeDL: op = str_op if not m: - raise SyntaxError('Invalid filter specification %r' % filter_spec) + raise SyntaxError(f'Invalid filter specification {filter_spec!r}') def _filter(f): actual_value = f.get(m.group('key')) @@ -2105,7 +2187,12 @@ class YoutubeDL: def _check_formats(self, formats): for f in formats: - self.to_screen('[info] Testing format %s' % f['format_id']) + working = f.get('__working') + if working is not None: + if working: + yield f + continue + self.to_screen('[info] Testing format {}'.format(f['format_id'])) path = self.get_output_path('temp') if not self._ensure_dir_exists(f'{path}/'): continue @@ -2113,41 +2200,51 @@ class YoutubeDL: temp_file.close() try: success, _ = self.dl(temp_file.name, f, test=True) - except (DownloadError, OSError, ValueError) + network_exceptions: + except (DownloadError, OSError, ValueError, *network_exceptions): success = False finally: if os.path.exists(temp_file.name): try: os.remove(temp_file.name) except OSError: - self.report_warning('Unable to delete temporary file "%s"' % temp_file.name) + self.report_warning(f'Unable to delete temporary file "{temp_file.name}"') + f['__working'] = success if success: yield f else: - self.to_screen('[info] Unable to download format %s. Skipping...' % f['format_id']) + self.to_screen('[info] Unable to download format {}. Skipping...'.format(f['format_id'])) - def _default_format_spec(self, info_dict, download=True): + def _select_formats(self, formats, selector): + return list(selector({ + 'formats': formats, + 'has_merged_format': any('none' not in (f.get('acodec'), f.get('vcodec')) for f in formats), + 'incomplete_formats': (all(f.get('vcodec') == 'none' for f in formats) # No formats with video + or all(f.get('acodec') == 'none' for f in formats)), # OR, No formats with audio + })) + + def _default_format_spec(self, info_dict): + prefer_best = ( + self.params['outtmpl']['default'] == '-' + or (info_dict.get('is_live') and not self.params.get('live_from_start'))) def can_merge(): merger = FFmpegMergerPP(self) return merger.available and merger.can_merge() - prefer_best = ( - not self.params.get('simulate') - and download - and ( - not can_merge() - or info_dict.get('is_live') and not self.params.get('live_from_start') - or self.params['outtmpl']['default'] == '-')) - compat = ( - prefer_best - or self.params.get('allow_multiple_audio_streams', False) - or 'format-spec' in self.params['compat_opts']) + if not prefer_best and not can_merge(): + prefer_best = True + formats = self._get_formats(info_dict) + evaluate_formats = lambda spec: self._select_formats(formats, self.build_format_selector(spec)) + if evaluate_formats('b/bv+ba') != evaluate_formats('bv*+ba/b'): + self.report_warning('ffmpeg not found. The downloaded format may not be the best available. ' + 'Installing ffmpeg is strongly recommended: https://github.com/yt-dlp/yt-dlp#dependencies') - return ( - 'best/bestvideo+bestaudio' if prefer_best - else 'bestvideo*+bestaudio/best' if not compat - else 'bestvideo+bestaudio/best') + compat = (self.params.get('allow_multiple_audio_streams') + or 'format-spec' in self.params['compat_opts']) + + return ('best/bestvideo+bestaudio' if prefer_best + else 'bestvideo+bestaudio/best' if compat + else 'bestvideo*+bestaudio/best') def build_format_selector(self, format_spec): def syntax_error(note, start): @@ -2167,8 +2264,8 @@ class YoutubeDL: def _parse_filter(tokens): filter_parts = [] - for type, string_, start, _, _ in tokens: - if type == tokenize.OP and string_ == ']': + for type_, string_, _start, _, _ in tokens: + if type_ == tokenize.OP and string_ == ']': return ''.join(filter_parts) else: filter_parts.append(string_) @@ -2178,23 +2275,23 @@ class YoutubeDL: # E.g. 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9' ALLOWED_OPS = ('/', '+', ',', '(', ')') last_string, last_start, last_end, last_line = None, None, None, None - for type, string_, start, end, line in tokens: - if type == tokenize.OP and string_ == '[': + for type_, string_, start, end, line in tokens: + if type_ == tokenize.OP and string_ == '[': if last_string: yield tokenize.NAME, last_string, last_start, last_end, last_line last_string = None - yield type, string_, start, end, line + yield type_, string_, start, end, line # everything inside brackets will be handled by _parse_filter - for type, string_, start, end, line in tokens: - yield type, string_, start, end, line - if type == tokenize.OP and string_ == ']': + for type_, string_, start, end, line in tokens: + yield type_, string_, start, end, line + if type_ == tokenize.OP and string_ == ']': break - elif type == tokenize.OP and string_ in ALLOWED_OPS: + elif type_ == tokenize.OP and string_ in ALLOWED_OPS: if last_string: yield tokenize.NAME, last_string, last_start, last_end, last_line last_string = None - yield type, string_, start, end, line - elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]: + yield type_, string_, start, end, line + elif type_ in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]: if not last_string: last_string = string_ last_start = start @@ -2207,13 +2304,13 @@ class YoutubeDL: def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False): selectors = [] current_selector = None - for type, string_, start, _, _ in tokens: - # ENCODING is only defined in python 3.x - if type == getattr(tokenize, 'ENCODING', None): + for type_, string_, start, _, _ in tokens: + # ENCODING is only defined in Python 3.x + if type_ == getattr(tokenize, 'ENCODING', None): continue - elif type in [tokenize.NAME, tokenize.NUMBER]: + elif type_ in [tokenize.NAME, tokenize.NUMBER]: current_selector = FormatSelector(SINGLE, string_, []) - elif type == tokenize.OP: + elif type_ == tokenize.OP: if string_ == ')': if not inside_group: # ')' will be handled by the parentheses group @@ -2256,7 +2353,7 @@ class YoutubeDL: current_selector = FormatSelector(MERGE, (selector_1, selector_2), []) else: raise syntax_error(f'Operator not recognized: "{string_}"', start) - elif type == tokenize.ENDMARKER: + elif type_ == tokenize.ENDMARKER: break if current_selector: selectors.append(current_selector) @@ -2297,7 +2394,7 @@ class YoutubeDL: vexts=[f['ext'] for f in video_fmts], aexts=[f['ext'] for f in audio_fmts], preferences=(try_call(lambda: self.params['merge_output_format'].split('/')) - or self.params.get('prefer_free_formats') and ('webm', 'mkv'))) + or (self.params.get('prefer_free_formats') and ('webm', 'mkv')))) filtered = lambda *keys: filter(None, (traverse_obj(fmt, *keys) for fmt in formats_info)) @@ -2331,7 +2428,7 @@ class YoutubeDL: 'acodec': the_only_audio.get('acodec'), 'abr': the_only_audio.get('abr'), 'asr': the_only_audio.get('asr'), - 'audio_channels': the_only_audio.get('audio_channels') + 'audio_channels': the_only_audio.get('audio_channels'), }) return new_dict @@ -2346,7 +2443,7 @@ class YoutubeDL: return for f in formats: - if f.get('has_drm'): + if f.get('has_drm') or f.get('__needs_testing'): yield from self._check_formats([f]) else: yield f @@ -2412,9 +2509,9 @@ class YoutubeDL: format_fallback = not format_type and not format_modified # for b, w _filter_f = ( - (lambda f: f.get('%scodec' % format_type) != 'none') + (lambda f: f.get(f'{format_type}codec') != 'none') if format_type and format_modified # bv*, ba*, wv*, wa* - else (lambda f: f.get('%scodec' % not_format_type) == 'none') + else (lambda f: f.get(f'{not_format_type}codec') == 'none') if format_type # bv, ba, wv, wa else (lambda f: f.get('vcodec') != 'none' and f.get('acodec') != 'none') if not format_modified # b, w @@ -2440,7 +2537,7 @@ class YoutubeDL: # for extractors with incomplete formats (audio only (soundcloud) # or video only (imgur)) best/worst will fallback to # best/worst {video,audio}-only format - matches = formats + matches = list(filter(lambda f: f.get('vcodec') != 'none' or f.get('acodec') != 'none', formats)) elif seperate_fallback and not ctx['has_merged_format']: # for compatibility with youtube-dl when there is no pre-merged format matches = list(filter(seperate_fallback, formats)) @@ -2459,9 +2556,16 @@ class YoutubeDL: return selector_function(ctx_copy) return final_selector - stream = io.BytesIO(format_spec.encode()) + # HACK: Python 3.12 changed the underlying parser, rendering '7_a' invalid + # Prefix numbers with random letters to avoid it being classified as a number + # See: https://github.com/yt-dlp/yt-dlp/pulls/8797 + # TODO: Implement parser not reliant on tokenize.tokenize + prefix = ''.join(random.choices(string.ascii_letters, k=32)) + stream = io.BytesIO(re.sub(r'\d[_\d]*', rf'{prefix}\g<0>', format_spec).encode()) try: - tokens = list(_remove_unused_ops(tokenize.tokenize(stream.readline))) + tokens = list(_remove_unused_ops( + token._replace(string=token.string.replace(prefix, '')) + for token in tokenize.tokenize(stream.readline))) except tokenize.TokenError: raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec))) @@ -2475,7 +2579,7 @@ class YoutubeDL: def __next__(self): if self.counter >= len(self.tokens): - raise StopIteration() + raise StopIteration value = self.tokens[self.counter] self.counter += 1 return value @@ -2558,7 +2662,7 @@ class YoutubeDL: self._sort_thumbnails(thumbnails) for i, t in enumerate(thumbnails): if t.get('id') is None: - t['id'] = '%d' % i + t['id'] = str(i) if t.get('width') and t.get('height'): t['resolution'] = '%dx%d' % (t['width'], t['height']) t['url'] = sanitize_url(t['url']) @@ -2591,9 +2695,12 @@ class YoutubeDL: # Working around out-of-range timestamp values (e.g. negative ones on Windows, # see http://bugs.python.org/issue1646728) with contextlib.suppress(ValueError, OverflowError, OSError): - upload_date = datetime.datetime.utcfromtimestamp(info_dict[ts_key]) + upload_date = dt.datetime.fromtimestamp(info_dict[ts_key], dt.timezone.utc) info_dict[date_key] = upload_date.strftime('%Y%m%d') + if not info_dict.get('release_year'): + info_dict['release_year'] = traverse_obj(info_dict, ('release_date', {lambda x: int(x[:4])})) + live_keys = ('is_live', 'was_live') live_status = info_dict.get('live_status') if live_status is None: @@ -2616,8 +2723,17 @@ class YoutubeDL: # Auto generate title fields corresponding to the *_number fields when missing # in order to always have clean titles. This is very common for TV series. for field in ('chapter', 'season', 'episode'): - if final and info_dict.get('%s_number' % field) is not None and not info_dict.get(field): - info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field]) + if final and info_dict.get(f'{field}_number') is not None and not info_dict.get(field): + info_dict[field] = '%s %d' % (field.capitalize(), info_dict[f'{field}_number']) + + for old_key, new_key in self._deprecated_multivalue_fields.items(): + if new_key in info_dict and old_key in info_dict: + if '_version' not in info_dict: # HACK: Do not warn when using --load-info-json + self.deprecation_warning(f'Do not return {old_key!r} when {new_key!r} is present') + elif old_value := info_dict.get(old_key): + info_dict[new_key] = old_value.split(', ') + elif new_value := info_dict.get(new_key): + info_dict[old_key] = ', '.join(v.replace(',', '\N{FULLWIDTH COMMA}') for v in new_value) def _raise_pending_errors(self, info): err = info.pop('__pending_error', None) @@ -2640,8 +2756,8 @@ class YoutubeDL: def report_force_conversion(field, field_not, conversion): self.report_warning( - '"%s" field is not %s - forcing %s conversion, there is an error in extractor' - % (field, field_not, conversion)) + f'"{field}" field is not {field_not} - forcing {conversion} conversion, ' + 'there is an error in extractor') def sanitize_string_field(info, string_field): field = info.get(string_field) @@ -2733,7 +2849,7 @@ class YoutubeDL: get_from_start = not info_dict.get('is_live') or bool(self.params.get('live_from_start')) if not get_from_start: - info_dict['title'] += ' ' + datetime.datetime.now().strftime('%Y-%m-%d %H:%M') + info_dict['title'] += ' ' + dt.datetime.now().strftime('%Y-%m-%d %H:%M') if info_dict.get('is_live') and formats: formats = [f for f in formats if bool(f.get('is_from_start')) == get_from_start] if get_from_start and not formats: @@ -2758,25 +2874,25 @@ class YoutubeDL: if not formats: self.raise_no_formats(info_dict) - for format in formats: - sanitize_string_field(format, 'format_id') - sanitize_numeric_fields(format) - format['url'] = sanitize_url(format['url']) - if format.get('ext') is None: - format['ext'] = determine_ext(format['url']).lower() - if format.get('protocol') is None: - format['protocol'] = determine_protocol(format) - if format.get('resolution') is None: - format['resolution'] = self.format_resolution(format, default=None) - if format.get('dynamic_range') is None and format.get('vcodec') != 'none': - format['dynamic_range'] = 'SDR' - if format.get('aspect_ratio') is None: - format['aspect_ratio'] = try_call(lambda: round(format['width'] / format['height'], 2)) - if (not format.get('manifest_url') # For fragmented formats, "tbr" is often max bitrate and not average - and info_dict.get('duration') and format.get('tbr') - and not format.get('filesize') and not format.get('filesize_approx')): - format['filesize_approx'] = int(info_dict['duration'] * format['tbr'] * (1024 / 8)) - format['http_headers'] = self._calc_headers(collections.ChainMap(format, info_dict), load_cookies=True) + for fmt in formats: + sanitize_string_field(fmt, 'format_id') + sanitize_numeric_fields(fmt) + fmt['url'] = sanitize_url(fmt['url']) + FormatSorter._fill_sorting_fields(fmt) + if fmt['ext'] in ('aac', 'opus', 'mp3', 'flac', 'vorbis'): + if fmt.get('acodec') is None: + fmt['acodec'] = fmt['ext'] + if fmt.get('resolution') is None: + fmt['resolution'] = self.format_resolution(fmt, default=None) + if fmt.get('dynamic_range') is None and fmt.get('vcodec') != 'none': + fmt['dynamic_range'] = 'SDR' + if fmt.get('aspect_ratio') is None: + fmt['aspect_ratio'] = try_call(lambda: round(fmt['width'] / fmt['height'], 2)) + # For fragmented formats, "tbr" is often max bitrate and not average + if (('manifest-filesize-approx' in self.params['compat_opts'] or not fmt.get('manifest_url')) + and not fmt.get('filesize') and not fmt.get('filesize_approx')): + fmt['filesize_approx'] = filesize_from_tbr(fmt.get('tbr'), info_dict.get('duration')) + fmt['http_headers'] = self._calc_headers(collections.ChainMap(fmt, info_dict), load_cookies=True) # Safeguard against old/insecure infojson when using --load-info-json if info_dict.get('http_headers'): @@ -2789,36 +2905,36 @@ class YoutubeDL: self.sort_formats({ 'formats': formats, - '_format_sort_fields': info_dict.get('_format_sort_fields') + '_format_sort_fields': info_dict.get('_format_sort_fields'), }) # Sanitize and group by format_id formats_dict = {} - for i, format in enumerate(formats): - if not format.get('format_id'): - format['format_id'] = str(i) + for i, fmt in enumerate(formats): + if not fmt.get('format_id'): + fmt['format_id'] = str(i) else: # Sanitize format_id from characters used in format selector expression - format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id']) - formats_dict.setdefault(format['format_id'], []).append(format) + fmt['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', fmt['format_id']) + formats_dict.setdefault(fmt['format_id'], []).append(fmt) # Make sure all formats have unique format_id common_exts = set(itertools.chain(*self._format_selection_exts.values())) for format_id, ambiguous_formats in formats_dict.items(): ambigious_id = len(ambiguous_formats) > 1 - for i, format in enumerate(ambiguous_formats): + for i, fmt in enumerate(ambiguous_formats): if ambigious_id: - format['format_id'] = '%s-%d' % (format_id, i) + fmt['format_id'] = f'{format_id}-{i}' # Ensure there is no conflict between id and ext in format selection # See https://github.com/yt-dlp/yt-dlp/issues/1282 - if format['format_id'] != format['ext'] and format['format_id'] in common_exts: - format['format_id'] = 'f%s' % format['format_id'] + if fmt['format_id'] != fmt['ext'] and fmt['format_id'] in common_exts: + fmt['format_id'] = 'f{}'.format(fmt['format_id']) - if format.get('format') is None: - format['format'] = '{id} - {res}{note}'.format( - id=format['format_id'], - res=self.format_resolution(format), - note=format_field(format, 'format_note', ' (%s)'), + if fmt.get('format') is None: + fmt['format'] = '{id} - {res}{note}'.format( + id=fmt['format_id'], + res=self.format_resolution(fmt), + note=format_field(fmt, 'format_note', ' (%s)'), ) if self.params.get('check_formats') is True: @@ -2871,16 +2987,11 @@ class YoutubeDL: continue if format_selector is None: - req_format = self._default_format_spec(info_dict, download=download) + req_format = self._default_format_spec(info_dict) self.write_debug(f'Default format spec: {req_format}') format_selector = self.build_format_selector(req_format) - formats_to_download = list(format_selector({ - 'formats': formats, - 'has_merged_format': any('none' not in (f.get('acodec'), f.get('vcodec')) for f in formats), - 'incomplete_formats': (all(f.get('vcodec') == 'none' for f in formats) # No formats with video - or all(f.get('acodec') == 'none' for f in formats)), # OR, No formats with audio - })) + formats_to_download = self._select_formats(formats, format_selector) if interactive_format_selection and not formats_to_download: self.report_error('Requested format is not available', tb=False, is_error=False) continue @@ -2945,7 +3056,7 @@ class YoutubeDL: info_dict['requested_downloads'] = downloaded_formats info_dict = self.run_all_pps('after_video', info_dict) if max_downloads_reached: - raise MaxDownloadsReached() + raise MaxDownloadsReached # We update the info dict with the selected best quality format (backwards compatibility) info_dict.update(best_format) @@ -3006,8 +3117,8 @@ class YoutubeDL: else: f = formats[-1] self.report_warning( - 'No subtitle format found matching "%s" for language %s, ' - 'using %s' % (formats_query, lang, f['ext'])) + 'No subtitle format found matching "{}" for language {}, ' + 'using {}. Use --list-subs for a list of available subtitles'.format(formats_query, lang, f['ext'])) subs[lang] = f return subs @@ -3086,11 +3197,12 @@ class YoutubeDL: if test: verbose = self.params.get('verbose') + quiet = self.params.get('quiet') or not verbose params = { 'test': True, - 'quiet': self.params.get('quiet') or not verbose, + 'quiet': quiet, 'verbose': verbose, - 'noprogress': not verbose, + 'noprogress': quiet, 'nopart': True, 'skip_unavailable_fragments': False, 'keep_fragments': False, @@ -3125,6 +3237,7 @@ class YoutubeDL: os.remove(file) return None + @_catch_unsafe_extension_error def process_info(self, info_dict): """Process a single resolved IE result. (Modifies it in-place)""" @@ -3162,7 +3275,7 @@ class YoutubeDL: def check_max_downloads(): if self._num_downloads >= float(self.params.get('max_downloads') or 'inf'): - raise MaxDownloadsReached() + raise MaxDownloadsReached if self.params.get('simulate'): info_dict['__write_download_archive'] = self.params.get('force_write_download_archive') @@ -3171,9 +3284,9 @@ class YoutubeDL: if full_filename is None: return - if not self._ensure_dir_exists(encodeFilename(full_filename)): + if not self._ensure_dir_exists(full_filename): return - if not self._ensure_dir_exists(encodeFilename(temp_filename)): + if not self._ensure_dir_exists(temp_filename): return if self._write_description('video', info_dict, @@ -3205,16 +3318,16 @@ class YoutubeDL: if self.params.get('writeannotations', False): annofn = self.prepare_filename(info_dict, 'annotation') if annofn: - if not self._ensure_dir_exists(encodeFilename(annofn)): + if not self._ensure_dir_exists(annofn): return - if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(annofn)): + if not self.params.get('overwrites', True) and os.path.exists(annofn): self.to_screen('[info] Video annotations are already present') elif not info_dict.get('annotations'): self.report_warning('There are no annotations to write.') else: try: self.to_screen('[info] Writing video annotations to: ' + annofn) - with open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile: + with open(annofn, 'w', encoding='utf-8') as annofile: annofile.write(info_dict['annotations']) except (KeyError, TypeError): self.report_warning('There are no annotations to write.') @@ -3230,14 +3343,14 @@ class YoutubeDL: f'Cannot write internet shortcut file because the actual URL of "{info_dict["webpage_url"]}" is unknown') return True linkfn = replace_extension(self.prepare_filename(info_dict, 'link'), link_type, info_dict.get('ext')) - if not self._ensure_dir_exists(encodeFilename(linkfn)): + if not self._ensure_dir_exists(linkfn): return False - if self.params.get('overwrites', True) and os.path.exists(encodeFilename(linkfn)): + if self.params.get('overwrites', True) and os.path.exists(linkfn): self.to_screen(f'[info] Internet shortcut (.{link_type}) is already present') return True try: self.to_screen(f'[info] Writing internet shortcut (.{link_type}) to: {linkfn}') - with open(encodeFilename(to_high_limit_path(linkfn)), 'w', encoding='utf-8', + with open(to_high_limit_path(linkfn), 'w', encoding='utf-8', newline='\r\n' if link_type == 'url' else '\n') as linkfile: template_vars = {'url': url} if link_type == 'desktop': @@ -3268,7 +3381,7 @@ class YoutubeDL: if self.params.get('skip_download'): info_dict['filepath'] = temp_filename - info_dict['__finaldir'] = os.path.dirname(os.path.abspath(encodeFilename(full_filename))) + info_dict['__finaldir'] = os.path.dirname(os.path.abspath(full_filename)) info_dict['__files_to_move'] = files_to_move replace_info_dict(self.run_pp(MoveFilesAfterDownloadPP(self, False), info_dict)) info_dict['__write_download_archive'] = self.params.get('force_write_download_archive') @@ -3336,7 +3449,7 @@ class YoutubeDL: for f in info_dict['requested_formats'] if fd != FFmpegFD else []: f['filepath'] = fname = prepend_extension( correct_ext(temp_filename, info_dict['ext']), - 'f%s' % f['format_id'], info_dict['ext']) + 'f{}'.format(f['format_id']), info_dict['ext']) downloaded.append(fname) info_dict['url'] = '\n'.join(f['url'] for f in info_dict['requested_formats']) success, real_download = self.dl(temp_filename, info_dict) @@ -3369,7 +3482,7 @@ class YoutubeDL: if temp_filename != '-': fname = prepend_extension( correct_ext(temp_filename, new_info['ext']), - 'f%s' % f['format_id'], new_info['ext']) + 'f{}'.format(f['format_id']), new_info['ext']) if not self._ensure_dir_exists(fname): return f['filepath'] = fname @@ -3398,14 +3511,14 @@ class YoutubeDL: self.report_file_already_downloaded(dl_filename) dl_filename = dl_filename or temp_filename - info_dict['__finaldir'] = os.path.dirname(os.path.abspath(encodeFilename(full_filename))) + info_dict['__finaldir'] = os.path.dirname(os.path.abspath(full_filename)) except network_exceptions as err: - self.report_error('unable to download video data: %s' % error_to_compat_str(err)) + self.report_error(f'unable to download video data: {err}') return except OSError as err: raise UnavailableVideoError(err) - except (ContentTooShortError, ) as err: + except ContentTooShortError as err: self.report_error(f'content too short (expected {err.expected} bytes and served {err.downloaded})') return @@ -3457,11 +3570,12 @@ class YoutubeDL: and info_dict.get('container') == 'm4a_dash', 'writing DASH m4a. Only some players support this container', FFmpegFixupM4aPP) - ffmpeg_fixup(downloader == 'hlsnative' and not self.params.get('hls_use_mpegts') - or info_dict.get('is_live') and self.params.get('hls_use_mpegts') is None, + ffmpeg_fixup((downloader == 'hlsnative' and not self.params.get('hls_use_mpegts')) + or (info_dict.get('is_live') and self.params.get('hls_use_mpegts') is None), 'Possible MPEG-TS in MP4 container or malformed AAC timestamps', FFmpegFixupM3u8PP) - ffmpeg_fixup(info_dict.get('is_live') and downloader == 'dashsegments', + ffmpeg_fixup(downloader == 'dashsegments' + and (info_dict.get('is_live') or info_dict.get('is_dash_periods')), 'Possible duplicate MOOV atoms', FFmpegFixupDuplicateMoovPP) ffmpeg_fixup(downloader == 'web_socket_fragment', 'Malformed timestamps detected', FFmpegFixupTimestampPP) @@ -3471,13 +3585,13 @@ class YoutubeDL: try: replace_info_dict(self.post_process(dl_filename, info_dict, files_to_move)) except PostProcessingError as err: - self.report_error('Postprocessing: %s' % str(err)) + self.report_error(f'Postprocessing: {err}') return try: for ph in self._post_hooks: ph(info_dict['filepath']) except Exception as err: - self.report_error('post hooks: %s' % str(err)) + self.report_error(f'post hooks: {err}') return info_dict['__write_download_archive'] = True @@ -3491,6 +3605,8 @@ class YoutubeDL: def wrapper(*args, **kwargs): try: res = func(*args, **kwargs) + except CookieLoadError: + raise except UnavailableVideoError as e: self.report_error(e) except DownloadCancelled as e: @@ -3538,11 +3654,13 @@ class YoutubeDL: raise self.report_warning(f'The info failed to download: {e}; trying with URL {webpage_url}') self.download([webpage_url]) + except ExtractorError as e: + self.report_error(e) return self._download_retcode @staticmethod def sanitize_info(info_dict, remove_private_keys=False): - ''' Sanitize the infodict for converting to json ''' + """ Sanitize the infodict for converting to json """ if info_dict is None: return info_dict info_dict.setdefault('epoch', int(time.time())) @@ -3551,14 +3669,14 @@ class YoutubeDL: 'version': __version__, 'current_git_head': current_git_head(), 'release_git_head': RELEASE_GIT_HEAD, - 'repository': REPOSITORY, + 'repository': ORIGIN, }) if remove_private_keys: reject = lambda k, v: v is None or k.startswith('__') or k in { 'requested_downloads', 'requested_formats', 'requested_subtitles', 'requested_entries', 'entries', 'filepath', '_filename', 'filename', 'infojson_filename', 'original_url', - 'playlist_autonumber', '_format_sort_fields', + 'playlist_autonumber', } else: reject = lambda k, v: False @@ -3577,7 +3695,7 @@ class YoutubeDL: @staticmethod def filter_requested_info(info_dict, actually_filter=True): - ''' Alias of sanitize_info for backward compatibility ''' + """ Alias of sanitize_info for backward compatibility """ return YoutubeDL.sanitize_info(info_dict, actually_filter) def _delete_downloaded_files(self, *files_to_delete, info={}, msg=None): @@ -3599,7 +3717,7 @@ class YoutubeDL: actual_post_extract(video_dict or {}) return - post_extractor = info_dict.pop('__post_extractor', None) or (lambda: {}) + post_extractor = info_dict.pop('__post_extractor', None) or dict info_dict.update(post_extractor()) actual_post_extract(info_dict or {}) @@ -3704,7 +3822,7 @@ class YoutubeDL: if format.get('width') and format.get('height'): return '%dx%d' % (format['width'], format['height']) elif format.get('height'): - return '%sp' % format['height'] + return '{}p'.format(format['height']) elif format.get('width'): return '%dx?' % format['width'] return default @@ -3721,7 +3839,7 @@ class YoutubeDL: if fdict.get('language'): if res: res += ' ' - res += '[%s]' % fdict['language'] + res += '[{}]'.format(fdict['language']) if fdict.get('format_note') is not None: if res: res += ' ' @@ -3733,7 +3851,7 @@ class YoutubeDL: if fdict.get('container') is not None: if res: res += ', ' - res += '%s container' % fdict['container'] + res += '{} container'.format(fdict['container']) if (fdict.get('vcodec') is not None and fdict.get('vcodec') != 'none'): if res: @@ -3748,7 +3866,7 @@ class YoutubeDL: if fdict.get('fps') is not None: if res: res += ', ' - res += '%sfps' % fdict['fps'] + res += '{}fps'.format(fdict['fps']) if fdict.get('acodec') is not None: if res: res += ', ' @@ -3791,7 +3909,7 @@ class YoutubeDL: format_field(f, 'format_id'), format_field(f, 'ext'), self.format_resolution(f), - self._format_note(f) + self._format_note(f), ] for f in formats if (f.get('preference') or 0) >= -1000] return render_table(['format code', 'extension', 'resolution', 'note'], table, extra_gap=1) @@ -3822,8 +3940,8 @@ class YoutubeDL: delim, ( format_field(f, 'filesize', ' \t%s', func=format_bytes) or format_field(f, 'filesize_approx', '≈\t%s', func=format_bytes) - or format_field(try_call(lambda: format_bytes(int(info_dict['duration'] * f['tbr'] * (1024 / 8)))), - None, self._format_out('~\t%s', self.Styles.SUPPRESS))), + or format_field(filesize_from_tbr(f.get('tbr'), info_dict.get('duration')), None, + self._format_out('~\t%s', self.Styles.SUPPRESS), func=format_bytes)), format_field(f, 'tbr', '\t%dk', func=round), shorten_protocol_name(f.get('protocol', '')), delim, @@ -3891,17 +4009,8 @@ class YoutubeDL: if not self.params.get('verbose'): return - from . import _IN_CLI # Must be delayed import - - # These imports can be slow. So import them only as needed - from .extractor.extractors import _LAZY_LOADER - from .extractor.extractors import ( - _PLUGIN_CLASSES as plugin_ies, - _PLUGIN_OVERRIDES as plugin_ie_overrides - ) - def get_encoding(stream): - ret = str(getattr(stream, 'encoding', 'missing (%s)' % type(stream).__name__)) + ret = str(getattr(stream, 'encoding', f'missing ({type(stream).__name__})')) additional_info = [] if os.environ.get('TERM', '').lower() == 'dumb': additional_info.append('dumb') @@ -3912,13 +4021,13 @@ class YoutubeDL: ret = f'{ret} ({",".join(additional_info)})' return ret - encoding_str = 'Encodings: locale %s, fs %s, pref %s, %s' % ( + encoding_str = 'Encodings: locale {}, fs {}, pref {}, {}'.format( locale.getpreferredencoding(), sys.getfilesystemencoding(), self.get_encoding(), ', '.join( f'{key} {get_encoding(stream)}' for key, stream in self._out_files.items_ - if stream is not None and key != 'console') + if stream is not None and key != 'console'), ) logger = self.params.get('logger') @@ -3934,23 +4043,24 @@ class YoutubeDL: source += '*' klass = type(self) write_debug(join_nonempty( - f'{"yt-dlp" if REPOSITORY == "yt-dlp/yt-dlp" else REPOSITORY} version', - f'{CHANNEL}@{__version__}', + f'{REPOSITORY.rpartition("/")[2]} version', + _make_label(ORIGIN, CHANNEL.partition('@')[2] or __version__, __version__), f'[{RELEASE_GIT_HEAD[:9]}]' if RELEASE_GIT_HEAD else '', '' if source == 'unknown' else f'({source})', - '' if _IN_CLI else 'API' if klass == YoutubeDL else f'API:{self.__module__}.{klass.__qualname__}', + '' if IN_CLI.value else 'API' if klass == YoutubeDL else f'API:{self.__module__}.{klass.__qualname__}', delim=' ')) - if not _IN_CLI: + if not IN_CLI.value: write_debug(f'params: {self.params}') - if not _LAZY_LOADER: - if os.environ.get('YTDLP_NO_LAZY_EXTRACTORS'): - write_debug('Lazy loading extractors is forcibly disabled') - else: - write_debug('Lazy loading extractors is disabled') + import_extractors() + lazy_extractors = LAZY_EXTRACTORS.value + if lazy_extractors is None: + write_debug('Lazy loading extractors is disabled') + elif not lazy_extractors: + write_debug('Lazy loading extractors is forcibly disabled') if self.params['compat_opts']: - write_debug('Compatibility options: %s' % ', '.join(self.params['compat_opts'])) + write_debug('Compatibility options: {}'.format(', '.join(self.params['compat_opts']))) if current_git_head(): write_debug(f'Git HEAD: {current_git_head()}') @@ -3959,14 +4069,14 @@ class YoutubeDL: exe_versions, ffmpeg_features = FFmpegPostProcessor.get_versions_and_features(self) ffmpeg_features = {key for key, val in ffmpeg_features.items() if val} if ffmpeg_features: - exe_versions['ffmpeg'] += ' (%s)' % ','.join(sorted(ffmpeg_features)) + exe_versions['ffmpeg'] += ' ({})'.format(','.join(sorted(ffmpeg_features))) exe_versions['rtmpdump'] = rtmpdump_version() exe_versions['phantomjs'] = PhantomJSwrapper._version() exe_str = ', '.join( f'{exe} {v}' for exe, v in sorted(exe_versions.items()) if v ) or 'none' - write_debug('exe versions: %s' % exe_str) + write_debug(f'exe versions: {exe_str}') from .compat.compat_utils import get_package_info from .dependencies import available_dependencies @@ -3976,33 +4086,28 @@ class YoutubeDL: })) or 'none')) write_debug(f'Proxy map: {self.proxies}') - # write_debug(f'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers.values())}') - for plugin_type, plugins in {'Extractor': plugin_ies, 'Post-Processor': plugin_pps}.items(): - display_list = ['%s%s' % ( - klass.__name__, '' if klass.__name__ == name else f' as {name}') - for name, klass in plugins.items()] + write_debug(f'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers.values())}') + + for plugin_type, plugins in (('Extractor', plugin_ies), ('Post-Processor', plugin_pps)): + display_list = [ + klass.__name__ if klass.__name__ == name else f'{klass.__name__} as {name}' + for name, klass in plugins.value.items()] if plugin_type == 'Extractor': display_list.extend(f'{plugins[-1].IE_NAME.partition("+")[2]} ({parent.__name__})' - for parent, plugins in plugin_ie_overrides.items()) + for parent, plugins in plugin_ies_overrides.value.items()) if not display_list: continue write_debug(f'{plugin_type} Plugins: {", ".join(sorted(display_list))}') - plugin_dirs = plugin_directories() - if plugin_dirs: - write_debug(f'Plugin directories: {plugin_dirs}') + plugin_dirs_msg = 'none' + if not plugin_dirs.value: + plugin_dirs_msg = 'none (disabled)' + else: + found_plugin_directories = plugin_directories() + if found_plugin_directories: + plugin_dirs_msg = ', '.join(found_plugin_directories) - # Not implemented - if False and self.params.get('call_home'): - ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode() - write_debug('Public IP address: %s' % ipaddr) - latest_version = self.urlopen( - 'https://yt-dl.org/latest/version').read().decode() - if version_tuple(latest_version) > version_tuple(__version__): - self.report_warning( - 'You are using an outdated version (newest version: %s)! ' - 'See https://yt-dl.org/update if you need help updating.' % - latest_version) + write_debug(f'Plugin directories: {plugin_dirs_msg}') @functools.cached_property def proxies(self): @@ -4023,8 +4128,14 @@ class YoutubeDL: @functools.cached_property def cookiejar(self): """Global cookiejar instance""" - return load_cookies( - self.params.get('cookiefile'), self.params.get('cookiesfrombrowser'), self) + try: + return load_cookies( + self.params.get('cookiefile'), self.params.get('cookiesfrombrowser'), self) + except CookieLoadError as error: + cause = error.__context__ + # compat: <=py3.9: `traceback.format_exception` has a different signature + self.report_error(str(cause), tb=''.join(traceback.format_exception(None, cause, cause.__traceback__))) + raise @property def _opener(self): @@ -4035,6 +4146,22 @@ class YoutubeDL: handler = self._request_director.handlers['Urllib'] return handler._get_instance(cookiejar=self.cookiejar, proxies=self.proxies) + def _get_available_impersonate_targets(self): + # TODO(future): make available as public API + return [ + (target, rh.RH_NAME) + for rh in self._request_director.handlers.values() + if isinstance(rh, ImpersonateRequestHandler) + for target in reversed(rh.supported_targets) + ] + + def _impersonate_target_available(self, target): + # TODO(future): make available as public API + return any( + rh.is_supported_target(target) + for rh in self._request_director.handlers.values() + if isinstance(rh, ImpersonateRequestHandler)) + def urlopen(self, req): """ Start an HTTP download """ if isinstance(req, str): @@ -4059,12 +4186,36 @@ class YoutubeDL: return self._request_director.send(req) except NoSupportingHandlers as e: for ue in e.unsupported_errors: + # FIXME: This depends on the order of errors. if not (ue.handler and ue.msg): continue if ue.handler.RH_KEY == 'Urllib' and 'unsupported url scheme: "file"' in ue.msg.lower(): raise RequestError( 'file:// URLs are disabled by default in yt-dlp for security reasons. ' 'Use --enable-file-urls to enable at your own risk.', cause=ue) from ue + if ( + 'unsupported proxy type: "https"' in ue.msg.lower() + and 'requests' not in self._request_director.handlers + and 'curl_cffi' not in self._request_director.handlers + ): + raise RequestError( + 'To use an HTTPS proxy for this request, one of the following dependencies needs to be installed: requests, curl_cffi') + + elif ( + re.match(r'unsupported url scheme: "wss?"', ue.msg.lower()) + and 'websockets' not in self._request_director.handlers + ): + raise RequestError( + 'This request requires WebSocket support. ' + 'Ensure one of the following dependencies are installed: websockets', + cause=ue) from ue + + elif re.match(r'unsupported (?:extensions: impersonate|impersonate target)', ue.msg.lower()): + raise RequestError( + f'Impersonate target "{req.extensions["impersonate"]}" is not available.' + f' See --list-impersonate-targets for available targets.' + f' This request requires browser impersonation, however you may be missing dependencies' + f' required to support this target.') raise except SSLError as e: if 'UNSAFE_LEGACY_RENEGOTIATION_DISABLED' in str(e): @@ -4074,8 +4225,6 @@ class YoutubeDL: 'SSLV3_ALERT_HANDSHAKE_FAILURE: The server may not support the current cipher list. ' 'Try using --legacy-server-connect', cause=e) from e raise - except HTTPError as e: # TODO: Remove in a future release - raise _CompatHTTPError(e) from e def build_request_director(self, handlers, preferences=None): logger = _YDLLogger(self) @@ -4099,6 +4248,7 @@ class YoutubeDL: 'timeout': 'socket_timeout', 'legacy_ssl_support': 'legacyserverconnect', 'enable_file_urls': 'enable_file_urls', + 'impersonate': 'impersonate', 'client_cert': { 'client_certificate': 'client_certificate', 'client_certificate_key': 'client_certificate_key', @@ -4107,8 +4257,14 @@ class YoutubeDL: }), )) director.preferences.update(preferences or []) + if 'prefer-legacy-http-handler' in self.params['compat_opts']: + director.preferences.add(lambda rh, _: 500 if rh.RH_KEY == 'Urllib' else 0) return director + @functools.cached_property + def _request_director(self): + return self.build_request_director(_REQUEST_HANDLERS.values(), _RH_PREFERENCES) + def encode(self, s): if isinstance(s, bytes): return s # Already encoded @@ -4126,7 +4282,7 @@ class YoutubeDL: return encoding def _write_info_json(self, label, ie_result, infofn, overwrite=None): - ''' Write infojson and returns True = written, 'exists' = Already exists, False = skip, None = error ''' + """ Write infojson and returns True = written, 'exists' = Already exists, False = skip, None = error """ if overwrite is None: overwrite = self.params.get('overwrites', True) if not self.params.get('writeinfojson'): @@ -4149,7 +4305,7 @@ class YoutubeDL: return None def _write_description(self, label, ie_result, descfn): - ''' Write description and returns True = written, False = skip, None = error ''' + """ Write description and returns True = written, False = skip, None = error """ if not self.params.get('writedescription'): return False elif not descfn: @@ -4165,7 +4321,7 @@ class YoutubeDL: else: try: self.to_screen(f'[info] Writing {label} description to: {descfn}') - with open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile: + with open(descfn, 'w', encoding='utf-8') as descfile: descfile.write(ie_result['description']) except OSError: self.report_error(f'Cannot write {label} description file {descfn}') @@ -4173,7 +4329,7 @@ class YoutubeDL: return True def _write_subtitles(self, info_dict, filename): - ''' Write subtitles to file and return list of (sub_filename, final_sub_filename); or None if error''' + """ Write subtitles to file and return list of (sub_filename, final_sub_filename); or None if error""" ret = [] subtitles = info_dict.get('requested_subtitles') if not (self.params.get('writesubtitles') or self.params.get('writeautomaticsub')): @@ -4219,7 +4375,7 @@ class YoutubeDL: self.dl(sub_filename, sub_copy, subtitle=True) sub_info['filepath'] = sub_filename ret.append((sub_filename, sub_filename_final)) - except (DownloadError, ExtractorError, IOError, OSError, ValueError) + network_exceptions as err: + except (DownloadError, ExtractorError, OSError, ValueError, *network_exceptions) as err: msg = f'Unable to download video subtitles for {sub_lang!r}: {err}' if self.params.get('ignoreerrors') is not True: # False or 'only_download' if not self.params.get('ignoreerrors'): @@ -4229,7 +4385,7 @@ class YoutubeDL: return ret def _write_thumbnails(self, label, info_dict, filename, thumb_filename_base=None): - ''' Write thumbnails to file and return list of (thumb_filename, final_thumb_filename) ''' + """ Write thumbnails to file and return list of (thumb_filename, final_thumb_filename); or None if error """ write_all = self.params.get('write_all_thumbnails', False) thumbnails, ret = [], [] if write_all or self.params.get('writethumbnail', False): @@ -4245,16 +4401,21 @@ class YoutubeDL: self.write_debug(f'Skipping writing {label} thumbnail') return ret + if thumbnails and not self._ensure_dir_exists(filename): + return None + for idx, t in list(enumerate(thumbnails))[::-1]: - thumb_ext = (f'{t["id"]}.' if multiple else '') + determine_ext(t['url'], 'jpg') + thumb_ext = t.get('ext') or determine_ext(t['url'], 'jpg') + if multiple: + thumb_ext = f'{t["id"]}.{thumb_ext}' thumb_display_id = f'{label} thumbnail {t["id"]}' thumb_filename = replace_extension(filename, thumb_ext, info_dict.get('ext')) thumb_filename_final = replace_extension(thumb_filename_base, thumb_ext, info_dict.get('ext')) existing_thumb = self.existing_file((thumb_filename_final, thumb_filename)) if existing_thumb: - self.to_screen('[info] %s is already present' % ( - thumb_display_id if multiple else f'{label} thumbnail').capitalize()) + self.to_screen('[info] {} is already present'.format(( + thumb_display_id if multiple else f'{label} thumbnail').capitalize())) t['filepath'] = existing_thumb ret.append((existing_thumb, thumb_filename_final)) else: @@ -4262,7 +4423,7 @@ class YoutubeDL: try: uf = self.urlopen(Request(t['url'], headers=t.get('http_headers', {}))) self.to_screen(f'[info] Writing {thumb_display_id} to: {thumb_filename}') - with open(encodeFilename(thumb_filename), 'wb') as thumbf: + with open(thumb_filename, 'wb') as thumbf: shutil.copyfileobj(uf, thumbf) ret.append((thumb_filename, thumb_filename_final)) t['filepath'] = thumb_filename diff --git a/plugins/youtube_download/yt_dlp/__init__.py b/plugins/youtube_download/yt_dlp/__init__.py index 991dbcda..714d9ad5 100644 --- a/plugins/youtube_download/yt_dlp/__init__.py +++ b/plugins/youtube_download/yt_dlp/__init__.py @@ -1,10 +1,10 @@ -try: - import contextvars # noqa: F401 -except Exception: - raise Exception( - f'You are using an unsupported version of Python. Only Python versions 3.7 and above are supported by yt-dlp') # noqa: F541 +import sys -__license__ = 'Public Domain' +if sys.version_info < (3, 9): + raise ImportError( + f'You are using an unsupported version of Python. Only Python versions 3.9 and above are supported by yt-dlp') # noqa: F541 + +__license__ = 'The Unlicense' import collections import getpass @@ -12,15 +12,16 @@ import itertools import optparse import os import re -import sys import traceback -from .compat import compat_shlex_quote -from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS +from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS, CookieLoadError from .downloader.external import get_external_downloader from .extractor import list_extractor_classes from .extractor.adobepass import MSO_INFO +from .networking.impersonate import ImpersonateTarget +from .globals import IN_CLI, plugin_dirs from .options import parseOpts +from .plugins import load_all_plugins as _load_all_plugins from .postprocessor import ( FFmpegExtractAudioPP, FFmpegMergerPP, @@ -43,12 +44,12 @@ from .utils import ( GeoUtils, PlaylistEntries, SameFileError, - decodeOption, download_range_func, expand_path, float_or_none, format_field, int_or_none, + join_nonempty, match_filter_func, parse_bytes, parse_duration, @@ -57,15 +58,15 @@ from .utils import ( read_stdin, render_table, setproctitle, + shell_quote, traverse_obj, variadic, write_string, ) from .utils.networking import std_headers +from .utils._utils import _UnsafeExtensionError from .YoutubeDL import YoutubeDL -_IN_CLI = False - def _exit(status=0, *args): for msg in args: @@ -74,14 +75,16 @@ def _exit(status=0, *args): def get_urls(urls, batchfile, verbose): - # Batch file verification + """ + @param verbose -1: quiet, 0: normal, 1: verbose + """ batch_urls = [] if batchfile is not None: try: batch_urls = read_batch_urls( - read_stdin('URLs') if batchfile == '-' + read_stdin(None if verbose == -1 else 'URLs') if batchfile == '-' else open(expand_path(batchfile), encoding='utf-8', errors='ignore')) - if verbose: + if verbose == 1: write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n') except OSError: _exit(f'ERROR: batch file {batchfile} could not be read') @@ -112,9 +115,9 @@ def print_extractor_information(opts, urls): ie.description(markdown=False, search_examples=_SEARCHES) for ie in list_extractor_classes(opts.age_limit) if ie.working() and ie.IE_DESC is not False) elif opts.ap_list_mso: - out = 'Supported TV Providers:\n%s\n' % render_table( + out = 'Supported TV Providers:\n{}\n'.format(render_table( ['mso', 'mso name'], - [[mso_id, mso_info['name']] for mso_id, mso_info in MSO_INFO.items()]) + [[mso_id, mso_info['name']] for mso_id, mso_info in MSO_INFO.items()])) else: return False write_string(out, out=sys.stdout) @@ -126,7 +129,7 @@ def set_compat_opts(opts): if name not in opts.compat_opts: return False opts.compat_opts.discard(name) - opts.compat_opts.update(['*%s' % name]) + opts.compat_opts.update([f'*{name}']) return True def set_default_compat(compat_name, opt_name, default=True, remove_compat=True): @@ -153,6 +156,9 @@ def set_compat_opts(opts): opts.embed_infojson = False if 'format-sort' in opts.compat_opts: opts.format_sort.extend(FormatSorter.ytdl_default) + elif 'prefer-vp9-sort' in opts.compat_opts: + opts.format_sort.extend(FormatSorter._prefer_vp9_sort) + _video_multistreams_set = set_default_compat('multistreams', 'allow_multiple_video_streams', False, remove_compat=False) _audio_multistreams_set = set_default_compat('multistreams', 'allow_multiple_audio_streams', False, remove_compat=False) if _video_multistreams_set is False and _audio_multistreams_set is False: @@ -219,7 +225,7 @@ def validate_options(opts): validate_minmax(opts.sleep_interval, opts.max_sleep_interval, 'sleep interval') if opts.wait_for_video is not None: - min_wait, max_wait, *_ = map(parse_duration, opts.wait_for_video.split('-', 1) + [None]) + min_wait, max_wait, *_ = map(parse_duration, [*opts.wait_for_video.split('-', 1), None]) validate(min_wait is not None and not (max_wait is None and '-' in opts.wait_for_video), 'time range to wait for video', opts.wait_for_video) validate_minmax(min_wait, max_wait, 'time range to wait for video') @@ -230,6 +236,11 @@ def validate_options(opts): validate_regex('format sorting', f, FormatSorter.regex) # Postprocessor formats + if opts.convertsubtitles == 'none': + opts.convertsubtitles = None + if opts.convertthumbnails == 'none': + opts.convertthumbnails = None + validate_regex('merge output format', opts.merge_output_format, r'({0})(/({0}))*'.format('|'.join(map(re.escape, FFmpegMergerPP.SUPPORTED_EXTS)))) validate_regex('audio format', opts.audioformat, FFmpegExtractAudioPP.FORMAT_RE) @@ -249,9 +260,11 @@ def validate_options(opts): elif value in ('inf', 'infinite'): return float('inf') try: - return int(value) + int_value = int(value) except (TypeError, ValueError): validate(False, f'{name} retry count', value) + validate_positive(f'{name} retry count', int_value) + return int_value opts.retries = parse_retries('download', opts.retries) opts.fragment_retries = parse_retries('fragment', opts.fragment_retries) @@ -261,9 +274,9 @@ def validate_options(opts): # Retry sleep function def parse_sleep_func(expr): NUMBER_RE = r'\d+(?:\.\d+)?' - op, start, limit, step, *_ = tuple(re.fullmatch( + op, start, limit, step, *_ = (*tuple(re.fullmatch( rf'(?:(linear|exp)=)?({NUMBER_RE})(?::({NUMBER_RE})?)?(?::({NUMBER_RE}))?', - expr.strip()).groups()) + (None, None) + expr.strip()).groups()), None, None) if op == 'exp': return lambda n: min(float(start) * (float(step or 2) ** n), float(limit or 'inf')) @@ -281,18 +294,20 @@ def validate_options(opts): raise ValueError(f'invalid {key} retry sleep expression {expr!r}') # Bytes - def validate_bytes(name, value): + def validate_bytes(name, value, strict_positive=False): if value is None: return None numeric_limit = parse_bytes(value) - validate(numeric_limit is not None, 'rate limit', value) + validate(numeric_limit is not None, name, value) + if strict_positive: + validate_positive(name, numeric_limit, True) return numeric_limit - opts.ratelimit = validate_bytes('rate limit', opts.ratelimit) + opts.ratelimit = validate_bytes('rate limit', opts.ratelimit, True) opts.throttledratelimit = validate_bytes('throttled rate limit', opts.throttledratelimit) opts.min_filesize = validate_bytes('min filesize', opts.min_filesize) opts.max_filesize = validate_bytes('max filesize', opts.max_filesize) - opts.buffersize = validate_bytes('buffer size', opts.buffersize) + opts.buffersize = validate_bytes('buffer size', opts.buffersize, True) opts.http_chunk_size = validate_bytes('http chunk size', opts.http_chunk_size) # Output templates @@ -387,16 +402,19 @@ def validate_options(opts): f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}') opts.cookiesfrombrowser = (browser_name, profile, keyring, container) + if opts.impersonate is not None: + opts.impersonate = ImpersonateTarget.from_str(opts.impersonate.lower()) + # MetadataParser def metadataparser_actions(f): if isinstance(f, str): - cmd = '--parse-metadata %s' % compat_shlex_quote(f) + cmd = f'--parse-metadata {shell_quote(f)}' try: actions = [MetadataFromFieldPP.to_action(f)] except Exception as err: raise ValueError(f'{cmd} is invalid; {err}') else: - cmd = '--replace-in-metadata %s' % ' '.join(map(compat_shlex_quote, f)) + cmd = f'--replace-in-metadata {shell_quote(f)}' actions = ((MetadataParserPP.Actions.REPLACE, x, *f[1:]) for x in f[0].split(',')) for action in actions: @@ -407,13 +425,17 @@ def validate_options(opts): yield action if opts.metafromtitle is not None: - opts.parse_metadata.setdefault('pre_process', []).append('title:%s' % opts.metafromtitle) + opts.parse_metadata.setdefault('pre_process', []).append(f'title:{opts.metafromtitle}') opts.parse_metadata = { k: list(itertools.chain(*map(metadataparser_actions, v))) for k, v in opts.parse_metadata.items() } # Other options + opts.plugin_dirs = opts.plugin_dirs + if opts.plugin_dirs is None: + opts.plugin_dirs = ['default'] + if opts.playlist_items is not None: try: tuple(PlaylistEntries.parse_playlist_items(opts.playlist_items)) @@ -460,7 +482,7 @@ def validate_options(opts): default_downloader = ed.get_basename() for policy in opts.color.values(): - if policy not in ('always', 'auto', 'no_color', 'never'): + if policy not in ('always', 'auto', 'auto-tty', 'no_color', 'no_color-tty', 'never'): raise ValueError(f'"{policy}" is not a valid color policy') warnings, deprecation_warnings = [], [] @@ -586,6 +608,13 @@ def validate_options(opts): if opts.ap_username is not None and opts.ap_password is None: opts.ap_password = getpass.getpass('Type TV provider account password and press [Return]: ') + # compat option changes global state destructively; only allow from cli + if 'allow-unsafe-ext' in opts.compat_opts: + warnings.append( + 'Using allow-unsafe-ext opens you up to potential attacks. ' + 'Use with great care!') + _UnsafeExtensionError.sanitize_extension = lambda x, prepend=False: x + return warnings, deprecation_warnings @@ -596,7 +625,7 @@ def get_postprocessors(opts): yield { 'key': 'MetadataParser', 'actions': actions, - 'when': when + 'when': when, } sponsorblock_query = opts.sponsorblock_mark | opts.sponsorblock_remove if sponsorblock_query: @@ -604,19 +633,19 @@ def get_postprocessors(opts): 'key': 'SponsorBlock', 'categories': sponsorblock_query, 'api': opts.sponsorblock_api, - 'when': 'after_filter' + 'when': 'after_filter', } if opts.convertsubtitles: yield { 'key': 'FFmpegSubtitlesConvertor', 'format': opts.convertsubtitles, - 'when': 'before_dl' + 'when': 'before_dl', } if opts.convertthumbnails: yield { 'key': 'FFmpegThumbnailsConvertor', 'format': opts.convertthumbnails, - 'when': 'before_dl' + 'when': 'before_dl', } if opts.extractaudio: yield { @@ -641,7 +670,7 @@ def get_postprocessors(opts): yield { 'key': 'FFmpegEmbedSubtitle', # already_have_subtitle = True prevents the file from being deleted after embedding - 'already_have_subtitle': opts.writesubtitles and keep_subs + 'already_have_subtitle': opts.writesubtitles and keep_subs, } if not opts.writeautomaticsub and keep_subs: opts.writesubtitles = True @@ -654,7 +683,7 @@ def get_postprocessors(opts): 'remove_sponsor_segments': opts.sponsorblock_remove, 'remove_ranges': opts.remove_ranges, 'sponsorblock_chapter_title': opts.sponsorblock_chapter_title, - 'force_keyframes': opts.force_keyframes_at_cuts + 'force_keyframes': opts.force_keyframes_at_cuts, } # FFmpegMetadataPP should be run after FFmpegVideoConvertorPP and # FFmpegExtractAudioPP as containers before conversion may not support @@ -688,7 +717,7 @@ def get_postprocessors(opts): yield { 'key': 'EmbedThumbnail', # already_have_thumbnail = True prevents the file from being deleted after embedding - 'already_have_thumbnail': opts.writethumbnail + 'already_have_thumbnail': opts.writethumbnail, } if not opts.writethumbnail: opts.writethumbnail = True @@ -722,7 +751,7 @@ ParsedOptions = collections.namedtuple('ParsedOptions', ('parser', 'options', 'u def parse_options(argv=None): """@returns ParsedOptions(parser, opts, urls, ydl_opts)""" parser, opts, urls = parseOpts(argv) - urls = get_urls(urls, opts.batchfile, opts.verbose) + urls = get_urls(urls, opts.batchfile, -1 if opts.quiet and not opts.verbose else opts.verbose) set_compat_opts(opts) try: @@ -735,7 +764,7 @@ def parse_options(argv=None): print_only = bool(opts.forceprint) and all(k not in opts.forceprint for k in POSTPROCESS_WHEN[3:]) any_getting = any(getattr(opts, k) for k in ( 'dumpjson', 'dump_single_json', 'getdescription', 'getduration', 'getfilename', - 'getformat', 'getid', 'getthumbnail', 'gettitle', 'geturl' + 'getformat', 'getid', 'getthumbnail', 'gettitle', 'geturl', )) if opts.quiet is None: opts.quiet = any_getting or opts.print_json or bool(opts.forceprint) @@ -830,6 +859,7 @@ def parse_options(argv=None): 'noprogress': opts.quiet if opts.noprogress is None else opts.noprogress, 'progress_with_newline': opts.progress_with_newline, 'progress_template': opts.progress_template, + 'progress_delta': opts.progress_delta, 'playliststart': opts.playliststart, 'playlistend': opts.playlistend, 'playlistreverse': opts.playlist_reverse, @@ -858,8 +888,8 @@ def parse_options(argv=None): 'listsubtitles': opts.listsubtitles, 'subtitlesformat': opts.subtitlesformat, 'subtitleslangs': opts.subtitleslangs, - 'matchtitle': decodeOption(opts.matchtitle), - 'rejecttitle': decodeOption(opts.rejecttitle), + 'matchtitle': opts.matchtitle, + 'rejecttitle': opts.rejecttitle, 'max_downloads': opts.max_downloads, 'prefer_free_formats': opts.prefer_free_formats, 'trim_file_name': opts.trim_file_name, @@ -910,6 +940,7 @@ def parse_options(argv=None): 'postprocessors': postprocessors, 'fixup': opts.fixup, 'source_address': opts.source_address, + 'impersonate': opts.impersonate, 'call_home': opts.call_home, 'sleep_interval_requests': opts.sleep_interval_requests, 'sleep_interval': opts.sleep_interval, @@ -959,6 +990,11 @@ def _real_main(argv=None): if opts.ffmpeg_location: FFmpegPostProcessor._ffmpeg_location.set(opts.ffmpeg_location) + # load all plugins into the global lookup + plugin_dirs.value = opts.plugin_dirs + if plugin_dirs.value: + _load_all_plugins() + with YoutubeDL(ydl_opts) as ydl: pre_process = opts.update_self or opts.rm_cachedir actual_use = all_urls or opts.load_info_filename @@ -979,11 +1015,68 @@ def _real_main(argv=None): traceback.print_exc() ydl._download_retcode = 100 + if opts.list_impersonate_targets: + + known_targets = [ + # List of simplified targets we know are supported, + # to help users know what dependencies may be required. + (ImpersonateTarget('chrome'), 'curl_cffi'), + (ImpersonateTarget('safari'), 'curl_cffi'), + (ImpersonateTarget('firefox'), 'curl_cffi>=0.10'), + (ImpersonateTarget('edge'), 'curl_cffi'), + ] + + available_targets = ydl._get_available_impersonate_targets() + + def make_row(target, handler): + return [ + join_nonempty(target.client.title(), target.version, delim='-') or '-', + join_nonempty((target.os or '').title(), target.os_version, delim='-') or '-', + handler, + ] + + rows = [make_row(target, handler) for target, handler in available_targets] + + for known_target, known_handler in known_targets: + if not any( + known_target in target and known_handler.startswith(handler) + for target, handler in available_targets + ): + rows.insert(0, [ + ydl._format_out(text, ydl.Styles.SUPPRESS) + for text in make_row(known_target, f'{known_handler} (unavailable)') + ]) + + ydl.to_screen('[info] Available impersonate targets') + ydl.to_stdout(render_table(['Client', 'OS', 'Source'], rows, extra_gap=2, delim='-')) + return + if not actual_use: if pre_process: return ydl._download_retcode - ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv) + args = sys.argv[1:] if argv is None else argv + ydl.warn_if_short_id(args) + + # Show a useful error message and wait for keypress if not launched from shell on Windows + if not args and os.name == 'nt' and getattr(sys, 'frozen', False): + import ctypes.wintypes + import msvcrt + + kernel32 = ctypes.WinDLL('Kernel32') + + buffer = (1 * ctypes.wintypes.DWORD)() + attached_processes = kernel32.GetConsoleProcessList(buffer, 1) + # If we only have a single process attached, then the executable was double clicked + # When using `pyinstaller` with `--onefile`, two processes get attached + is_onefile = hasattr(sys, '_MEIPASS') and os.path.basename(sys._MEIPASS).startswith('_MEI') + if attached_processes == 1 or (is_onefile and attached_processes == 2): + print(parser._generate_error_message( + 'Do not double-click the executable, instead call it from a command line.\n' + 'Please read the README for further information on how to use yt-dlp: ' + 'https://github.com/yt-dlp/yt-dlp#readme')) + msvcrt.getch() + _exit(2) parser.error( 'You must provide at least one URL.\n' 'Type yt-dlp --help to see a list of all options.') @@ -1002,11 +1095,10 @@ def _real_main(argv=None): def main(argv=None): - global _IN_CLI - _IN_CLI = True + IN_CLI.value = True try: _exit(*variadic(_real_main(argv))) - except DownloadError: + except (CookieLoadError, DownloadError): _exit(1) except SameFileError as e: _exit(f'ERROR: {e}') @@ -1024,9 +1116,9 @@ def main(argv=None): from .extractor import gen_extractors, list_extractors __all__ = [ - 'main', 'YoutubeDL', - 'parse_options', 'gen_extractors', 'list_extractors', + 'main', + 'parse_options', ] diff --git a/plugins/youtube_download/yt_dlp/__main__.py b/plugins/youtube_download/yt_dlp/__main__.py index 78701df8..06c39203 100644 --- a/plugins/youtube_download/yt_dlp/__main__.py +++ b/plugins/youtube_download/yt_dlp/__main__.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Execute with -# $ python -m yt_dlp +# $ python3 -m yt_dlp import sys diff --git a/plugins/youtube_download/yt_dlp/__pyinstaller/hook-yt_dlp.py b/plugins/youtube_download/yt_dlp/__pyinstaller/hook-yt_dlp.py index 88c2b8b2..8e7f42f5 100644 --- a/plugins/youtube_download/yt_dlp/__pyinstaller/hook-yt_dlp.py +++ b/plugins/youtube_download/yt_dlp/__pyinstaller/hook-yt_dlp.py @@ -1,6 +1,6 @@ import sys -from PyInstaller.utils.hooks import collect_submodules +from PyInstaller.utils.hooks import collect_submodules, collect_data_files def pycryptodome_module(): @@ -10,7 +10,7 @@ def pycryptodome_module(): try: import Crypto # noqa: F401 print('WARNING: Using Crypto since Cryptodome is not available. ' - 'Install with: pip install pycryptodomex', file=sys.stderr) + 'Install with: python3 -m pip install pycryptodomex', file=sys.stderr) return 'Crypto' except ImportError: pass @@ -21,12 +21,16 @@ def get_hidden_imports(): yield from ('yt_dlp.compat._legacy', 'yt_dlp.compat._deprecated') yield from ('yt_dlp.utils._legacy', 'yt_dlp.utils._deprecated') yield pycryptodome_module() - yield from collect_submodules('websockets') + # Only `websockets` is required, others are collected just in case + for module in ('websockets', 'requests', 'urllib3'): + yield from collect_submodules(module) # These are auto-detected, but explicitly add them just in case - yield from ('mutagen', 'brotli', 'certifi') + yield from ('mutagen', 'brotli', 'certifi', 'secretstorage', 'curl_cffi') hiddenimports = list(get_hidden_imports()) print(f'Adding imports: {hiddenimports}') -excludedimports = ['youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts'] +excludedimports = ['youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts', 'bundle'] + +datas = collect_data_files('curl_cffi', includes=['cacert.pem']) diff --git a/plugins/youtube_download/yt_dlp/aes.py b/plugins/youtube_download/yt_dlp/aes.py index b3a383cd..065901d6 100644 --- a/plugins/youtube_download/yt_dlp/aes.py +++ b/plugins/youtube_download/yt_dlp/aes.py @@ -3,7 +3,6 @@ from math import ceil from .compat import compat_ord from .dependencies import Cryptodome -from .utils import bytes_to_intlist, intlist_to_bytes if Cryptodome.AES: def aes_cbc_decrypt_bytes(data, key, iv): @@ -17,15 +16,15 @@ if Cryptodome.AES: else: def aes_cbc_decrypt_bytes(data, key, iv): """ Decrypt bytes with AES-CBC using native implementation since pycryptodome is unavailable """ - return intlist_to_bytes(aes_cbc_decrypt(*map(bytes_to_intlist, (data, key, iv)))) + return bytes(aes_cbc_decrypt(*map(list, (data, key, iv)))) def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce): """ Decrypt bytes with AES-GCM using native implementation since pycryptodome is unavailable """ - return intlist_to_bytes(aes_gcm_decrypt_and_verify(*map(bytes_to_intlist, (data, key, tag, nonce)))) + return bytes(aes_gcm_decrypt_and_verify(*map(list, (data, key, tag, nonce)))) def aes_cbc_encrypt_bytes(data, key, iv, **kwargs): - return intlist_to_bytes(aes_cbc_encrypt(*map(bytes_to_intlist, (data, key, iv)), **kwargs)) + return bytes(aes_cbc_encrypt(*map(list, (data, key, iv)), **kwargs)) BLOCK_SIZE_BYTES = 16 @@ -68,7 +67,7 @@ def pad_block(block, padding_mode): raise NotImplementedError(f'Padding mode {padding_mode} is not implemented') if padding_mode == 'iso7816' and padding_size: - block = block + [0x80] # NB: += mutates list + block = [*block, 0x80] # NB: += mutates list padding_size -= 1 return block + [PADDING_BYTE[padding_mode]] * padding_size @@ -84,7 +83,7 @@ def aes_ecb_encrypt(data, key, iv=None): @returns {int[]} encrypted data """ expanded_key = key_expansion(key) - block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES)) + block_count = ceil(len(data) / BLOCK_SIZE_BYTES) encrypted_data = [] for i in range(block_count): @@ -104,15 +103,13 @@ def aes_ecb_decrypt(data, key, iv=None): @returns {int[]} decrypted data """ expanded_key = key_expansion(key) - block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES)) + block_count = ceil(len(data) / BLOCK_SIZE_BYTES) encrypted_data = [] for i in range(block_count): block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES] encrypted_data += aes_decrypt(block, expanded_key) - encrypted_data = encrypted_data[:len(data)] - - return encrypted_data + return encrypted_data[:len(data)] def aes_ctr_decrypt(data, key, iv): @@ -137,7 +134,7 @@ def aes_ctr_encrypt(data, key, iv): @returns {int[]} encrypted data """ expanded_key = key_expansion(key) - block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES)) + block_count = ceil(len(data) / BLOCK_SIZE_BYTES) counter = iter_vector(iv) encrypted_data = [] @@ -148,9 +145,7 @@ def aes_ctr_encrypt(data, key, iv): cipher_counter_block = aes_encrypt(counter_block, expanded_key) encrypted_data += xor(block, cipher_counter_block) - encrypted_data = encrypted_data[:len(data)] - - return encrypted_data + return encrypted_data[:len(data)] def aes_cbc_decrypt(data, key, iv): @@ -163,7 +158,7 @@ def aes_cbc_decrypt(data, key, iv): @returns {int[]} decrypted data """ expanded_key = key_expansion(key) - block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES)) + block_count = ceil(len(data) / BLOCK_SIZE_BYTES) decrypted_data = [] previous_cipher_block = iv @@ -174,9 +169,7 @@ def aes_cbc_decrypt(data, key, iv): decrypted_block = aes_decrypt(block, expanded_key) decrypted_data += xor(decrypted_block, previous_cipher_block) previous_cipher_block = block - decrypted_data = decrypted_data[:len(data)] - - return decrypted_data + return decrypted_data[:len(data)] def aes_cbc_encrypt(data, key, iv, *, padding_mode='pkcs7'): @@ -190,7 +183,7 @@ def aes_cbc_encrypt(data, key, iv, *, padding_mode='pkcs7'): @returns {int[]} encrypted data """ expanded_key = key_expansion(key) - block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES)) + block_count = ceil(len(data) / BLOCK_SIZE_BYTES) encrypted_data = [] previous_cipher_block = iv @@ -224,10 +217,10 @@ def aes_gcm_decrypt_and_verify(data, key, tag, nonce): hash_subkey = aes_encrypt([0] * BLOCK_SIZE_BYTES, key_expansion(key)) if len(nonce) == 12: - j0 = nonce + [0, 0, 0, 1] + j0 = [*nonce, 0, 0, 0, 1] else: fill = (BLOCK_SIZE_BYTES - (len(nonce) % BLOCK_SIZE_BYTES)) % BLOCK_SIZE_BYTES + 8 - ghash_in = nonce + [0] * fill + bytes_to_intlist((8 * len(nonce)).to_bytes(8, 'big')) + ghash_in = nonce + [0] * fill + list((8 * len(nonce)).to_bytes(8, 'big')) j0 = ghash(hash_subkey, ghash_in) # TODO: add nonce support to aes_ctr_decrypt @@ -236,17 +229,17 @@ def aes_gcm_decrypt_and_verify(data, key, tag, nonce): iv_ctr = inc(j0) decrypted_data = aes_ctr_decrypt(data, key, iv_ctr + [0] * (BLOCK_SIZE_BYTES - len(iv_ctr))) - pad_len = len(data) // 16 * 16 + pad_len = (BLOCK_SIZE_BYTES - (len(data) % BLOCK_SIZE_BYTES)) % BLOCK_SIZE_BYTES s_tag = ghash( hash_subkey, data - + [0] * (BLOCK_SIZE_BYTES - len(data) + pad_len) # pad - + bytes_to_intlist((0 * 8).to_bytes(8, 'big') # length of associated data - + ((len(data) * 8).to_bytes(8, 'big'))) # length of data + + [0] * pad_len # pad + + list((0 * 8).to_bytes(8, 'big') # length of associated data + + ((len(data) * 8).to_bytes(8, 'big'))), # length of data ) if tag != aes_ctr_encrypt(s_tag, key, j0): - raise ValueError("Mismatching authentication tag") + raise ValueError('Mismatching authentication tag') return decrypted_data @@ -288,9 +281,7 @@ def aes_decrypt(data, expanded_key): data = list(iter_mix_columns(data, MIX_COLUMN_MATRIX_INV)) data = shift_rows_inv(data) data = sub_bytes_inv(data) - data = xor(data, expanded_key[:BLOCK_SIZE_BYTES]) - - return data + return xor(data, expanded_key[:BLOCK_SIZE_BYTES]) def aes_decrypt_text(data, password, key_size_bytes): @@ -308,8 +299,8 @@ def aes_decrypt_text(data, password, key_size_bytes): """ NONCE_LENGTH_BYTES = 8 - data = bytes_to_intlist(base64.b64decode(data)) - password = bytes_to_intlist(password.encode()) + data = list(base64.b64decode(data)) + password = list(password.encode()) key = password[:key_size_bytes] + [0] * (key_size_bytes - len(password)) key = aes_encrypt(key[:BLOCK_SIZE_BYTES], key_expansion(key)) * (key_size_bytes // BLOCK_SIZE_BYTES) @@ -318,9 +309,7 @@ def aes_decrypt_text(data, password, key_size_bytes): cipher = data[NONCE_LENGTH_BYTES:] decrypted_data = aes_ctr_decrypt(cipher, key, nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)) - plaintext = intlist_to_bytes(decrypted_data) - - return plaintext + return bytes(decrypted_data) RCON = (0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36) @@ -428,9 +417,7 @@ def key_expansion(data): for _ in range(3 if key_size_bytes == 32 else 2 if key_size_bytes == 24 else 0): temp = data[-4:] data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes]) - data = data[:expanded_key_size_bytes] - - return data + return data[:expanded_key_size_bytes] def iter_vector(iv): @@ -511,7 +498,7 @@ def block_product(block_x, block_y): # NIST SP 800-38D, Algorithm 1 if len(block_x) != BLOCK_SIZE_BYTES or len(block_y) != BLOCK_SIZE_BYTES: - raise ValueError("Length of blocks need to be %d bytes" % BLOCK_SIZE_BYTES) + raise ValueError(f'Length of blocks need to be {BLOCK_SIZE_BYTES} bytes') block_r = [0xE1] + [0] * (BLOCK_SIZE_BYTES - 1) block_v = block_y[:] @@ -534,7 +521,7 @@ def ghash(subkey, data): # NIST SP 800-38D, Algorithm 2 if len(data) % BLOCK_SIZE_BYTES: - raise ValueError("Length of data should be %d bytes" % BLOCK_SIZE_BYTES) + raise ValueError(f'Length of data should be {BLOCK_SIZE_BYTES} bytes') last_y = [0] * BLOCK_SIZE_BYTES for i in range(0, len(data), BLOCK_SIZE_BYTES): @@ -547,19 +534,17 @@ def ghash(subkey, data): __all__ = [ 'aes_cbc_decrypt', 'aes_cbc_decrypt_bytes', - 'aes_ctr_decrypt', - 'aes_decrypt_text', - 'aes_decrypt', - 'aes_ecb_decrypt', - 'aes_gcm_decrypt_and_verify', - 'aes_gcm_decrypt_and_verify_bytes', - 'aes_cbc_encrypt', 'aes_cbc_encrypt_bytes', + 'aes_ctr_decrypt', 'aes_ctr_encrypt', + 'aes_decrypt', + 'aes_decrypt_text', + 'aes_ecb_decrypt', 'aes_ecb_encrypt', 'aes_encrypt', - + 'aes_gcm_decrypt_and_verify', + 'aes_gcm_decrypt_and_verify_bytes', 'key_expansion', 'pad_block', 'pkcs7_padding', diff --git a/plugins/youtube_download/yt_dlp/cache.py b/plugins/youtube_download/yt_dlp/cache.py index 9dd4f2f2..71dca82b 100644 --- a/plugins/youtube_download/yt_dlp/cache.py +++ b/plugins/youtube_download/yt_dlp/cache.py @@ -81,10 +81,10 @@ class Cache: cachedir = self._get_root_dir() if not any((term in cachedir) for term in ('cache', 'tmp')): - raise Exception('Not removing directory %s - this does not look like a cache dir' % cachedir) + raise Exception(f'Not removing directory {cachedir} - this does not look like a cache dir') self._ydl.to_screen( - 'Removing cache dir %s .' % cachedir, skip_eol=True) + f'Removing cache dir {cachedir} .', skip_eol=True) if os.path.exists(cachedir): self._ydl.to_screen('.', skip_eol=True) shutil.rmtree(cachedir) diff --git a/plugins/youtube_download/yt_dlp/casefold.py b/plugins/youtube_download/yt_dlp/casefold.py deleted file mode 100644 index 41a53e5b..00000000 --- a/plugins/youtube_download/yt_dlp/casefold.py +++ /dev/null @@ -1,5 +0,0 @@ -import warnings - -warnings.warn(DeprecationWarning(f'{__name__} is deprecated')) - -casefold = str.casefold diff --git a/plugins/youtube_download/yt_dlp/compat/__init__.py b/plugins/youtube_download/yt_dlp/compat/__init__.py index 832a9138..d7796206 100644 --- a/plugins/youtube_download/yt_dlp/compat/__init__.py +++ b/plugins/youtube_download/yt_dlp/compat/__init__.py @@ -1,5 +1,4 @@ import os -import sys import xml.etree.ElementTree as etree from .compat_utils import passthrough_module @@ -24,36 +23,14 @@ def compat_etree_fromstring(text): return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder())) -compat_os_name = os._name if os.name == 'java' else os.name - - -if compat_os_name == 'nt': - def compat_shlex_quote(s): - import re - return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"') -else: - from shlex import quote as compat_shlex_quote # noqa: F401 - - def compat_ord(c): return c if isinstance(c, int) else ord(c) -if compat_os_name == 'nt' and sys.version_info < (3, 8): - # os.path.realpath on Windows does not follow symbolic links - # prior to Python 3.8 (see https://bugs.python.org/issue9949) - def compat_realpath(path): - while os.path.islink(path): - path = os.path.abspath(os.readlink(path)) - return os.path.realpath(path) -else: - compat_realpath = os.path.realpath - - # Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl # See https://github.com/yt-dlp/yt-dlp/issues/792 # https://docs.python.org/3/library/os.path.html#os.path.expanduser -if compat_os_name in ('nt', 'ce'): +if os.name in ('nt', 'ce'): def compat_expanduser(path): HOME = os.environ.get('HOME') if not HOME: diff --git a/plugins/youtube_download/yt_dlp/compat/_deprecated.py b/plugins/youtube_download/yt_dlp/compat/_deprecated.py index 607bae99..445acc1a 100644 --- a/plugins/youtube_download/yt_dlp/compat/_deprecated.py +++ b/plugins/youtube_download/yt_dlp/compat/_deprecated.py @@ -8,16 +8,14 @@ passthrough_module(__name__, '.._legacy', callback=lambda attr: warnings.warn( DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=6)) del passthrough_module -import base64 -import urllib.error -import urllib.parse +import functools # noqa: F401 +import os -compat_str = str -compat_b64decode = base64.b64decode +compat_os_name = os.name +compat_realpath = os.path.realpath -compat_urlparse = urllib.parse -compat_parse_qs = urllib.parse.parse_qs -compat_urllib_parse_unquote = urllib.parse.unquote -compat_urllib_parse_urlencode = urllib.parse.urlencode -compat_urllib_parse_urlparse = urllib.parse.urlparse + +def compat_shlex_quote(s): + from ..utils import shell_quote + return shell_quote(s) diff --git a/plugins/youtube_download/yt_dlp/compat/_legacy.py b/plugins/youtube_download/yt_dlp/compat/_legacy.py index 90ccf0f1..dae2c145 100644 --- a/plugins/youtube_download/yt_dlp/compat/_legacy.py +++ b/plugins/youtube_download/yt_dlp/compat/_legacy.py @@ -30,11 +30,12 @@ from asyncio import run as compat_asyncio_run # noqa: F401 from re import Pattern as compat_Pattern # noqa: F401 from re import match as compat_Match # noqa: F401 -from . import compat_expanduser, compat_HTMLParseError, compat_realpath +from . import compat_expanduser, compat_HTMLParseError from .compat_utils import passthrough_module from ..dependencies import brotli as compat_brotli # noqa: F401 from ..dependencies import websockets as compat_websockets # noqa: F401 from ..dependencies.Cryptodome import AES as compat_pycrypto_AES # noqa: F401 +from ..networking.exceptions import HTTPError as compat_HTTPError passthrough_module(__name__, '...utils', ('WINDOWS_VT_MODE', 'windows_enable_vt_mode')) @@ -70,7 +71,6 @@ compat_html_parser_HTMLParseError = compat_HTMLParseError compat_HTMLParser = compat_html_parser_HTMLParser = html.parser.HTMLParser compat_http_client = http.client compat_http_server = http.server -compat_HTTPError = urllib.error.HTTPError compat_input = input compat_integer_types = (int, ) compat_itertools_count = itertools.count @@ -78,7 +78,7 @@ compat_kwargs = lambda kwargs: kwargs compat_map = map compat_numeric_types = (int, float, complex) compat_os_path_expanduser = compat_expanduser -compat_os_path_realpath = compat_realpath +compat_os_path_realpath = os.path.realpath compat_print = print compat_shlex_split = shlex.split compat_socket_create_connection = socket.create_connection @@ -88,7 +88,7 @@ compat_struct_unpack = struct.unpack compat_subprocess_get_DEVNULL = lambda: subprocess.DEVNULL compat_tokenize_tokenize = tokenize.tokenize compat_urllib_error = urllib.error -compat_urllib_HTTPError = urllib.error.HTTPError +compat_urllib_HTTPError = compat_HTTPError compat_urllib_parse = urllib.parse compat_urllib_parse_parse_qs = urllib.parse.parse_qs compat_urllib_parse_quote = urllib.parse.quote @@ -104,5 +104,12 @@ compat_xml_parse_error = compat_xml_etree_ElementTree_ParseError = etree.ParseEr compat_xpath = lambda xpath: xpath compat_zip = zip workaround_optparse_bug9161 = lambda: None +compat_str = str +compat_b64decode = base64.b64decode +compat_urlparse = urllib.parse +compat_parse_qs = urllib.parse.parse_qs +compat_urllib_parse_unquote = urllib.parse.unquote +compat_urllib_parse_urlencode = urllib.parse.urlencode +compat_urllib_parse_urlparse = urllib.parse.urlparse legacy = [] diff --git a/plugins/youtube_download/yt_dlp/compat/compat_utils.py b/plugins/youtube_download/yt_dlp/compat/compat_utils.py index 3ca46d27..d8b3c45c 100644 --- a/plugins/youtube_download/yt_dlp/compat/compat_utils.py +++ b/plugins/youtube_download/yt_dlp/compat/compat_utils.py @@ -15,7 +15,7 @@ def get_package_info(module): name=getattr(module, '_yt_dlp__identifier', module.__name__), version=str(next(filter(None, ( getattr(module, attr, None) - for attr in ('__version__', 'version_string', 'version') + for attr in ('_yt_dlp__version', '__version__', 'version_string', 'version') )), None))) @@ -57,7 +57,7 @@ def passthrough_module(parent, child, allowed_attributes=(..., ), *, callback=la callback(attr) return ret - @functools.lru_cache(maxsize=None) + @functools.cache def from_child(attr): nonlocal child if attr not in allowed_attributes: diff --git a/plugins/youtube_download/yt_dlp/compat/functools.py b/plugins/youtube_download/yt_dlp/compat/functools.py deleted file mode 100644 index ec003ea9..00000000 --- a/plugins/youtube_download/yt_dlp/compat/functools.py +++ /dev/null @@ -1,26 +0,0 @@ -# flake8: noqa: F405 -from functools import * # noqa: F403 - -from .compat_utils import passthrough_module - -passthrough_module(__name__, 'functools') -del passthrough_module - -try: - cache # >= 3.9 -except NameError: - cache = lru_cache(maxsize=None) - -try: - cached_property # >= 3.8 -except NameError: - class cached_property: - def __init__(self, func): - update_wrapper(self, func) - self.func = func - - def __get__(self, instance, _): - if instance is None: - return self - setattr(instance, self.func.__name__, self.func(instance)) - return getattr(instance, self.func.__name__) diff --git a/plugins/youtube_download/yt_dlp/compat/imghdr.py b/plugins/youtube_download/yt_dlp/compat/imghdr.py index 5d64ab07..4ae173fd 100644 --- a/plugins/youtube_download/yt_dlp/compat/imghdr.py +++ b/plugins/youtube_download/yt_dlp/compat/imghdr.py @@ -1,16 +1,22 @@ -tests = { - 'webp': lambda h: h[0:4] == b'RIFF' and h[8:] == b'WEBP', - 'png': lambda h: h[:8] == b'\211PNG\r\n\032\n', - 'jpeg': lambda h: h[6:10] in (b'JFIF', b'Exif'), - 'gif': lambda h: h[:6] in (b'GIF87a', b'GIF89a'), -} - - def what(file=None, h=None): """Detect format of image (Currently supports jpeg, png, webp, gif only) - Ref: https://github.com/python/cpython/blob/3.10/Lib/imghdr.py + Ref: https://github.com/python/cpython/blob/3.11/Lib/imghdr.py + Ref: https://www.w3.org/Graphics/JPEG/itu-t81.pdf """ if h is None: with open(file, 'rb') as f: h = f.read(12) - return next((type_ for type_, test in tests.items() if test(h)), None) + + if h.startswith(b'RIFF') and h.startswith(b'WEBP', 8): + return 'webp' + + if h.startswith(b'\x89PNG'): + return 'png' + + if h.startswith(b'\xFF\xD8\xFF'): + return 'jpeg' + + if h.startswith(b'GIF'): + return 'gif' + + return None diff --git a/plugins/youtube_download/yt_dlp/compat/urllib/__init__.py b/plugins/youtube_download/yt_dlp/compat/urllib/__init__.py index b27cc613..9084b3c2 100644 --- a/plugins/youtube_download/yt_dlp/compat/urllib/__init__.py +++ b/plugins/youtube_download/yt_dlp/compat/urllib/__init__.py @@ -1,7 +1,7 @@ # flake8: noqa: F405 from urllib import * # noqa: F403 -del request +del request # noqa: F821 from . import request # noqa: F401 from ..compat_utils import passthrough_module diff --git a/plugins/youtube_download/yt_dlp/compat/urllib/request.py b/plugins/youtube_download/yt_dlp/compat/urllib/request.py index ff63b2f0..dfc7f4a2 100644 --- a/plugins/youtube_download/yt_dlp/compat/urllib/request.py +++ b/plugins/youtube_download/yt_dlp/compat/urllib/request.py @@ -7,13 +7,13 @@ passthrough_module(__name__, 'urllib.request') del passthrough_module -from .. import compat_os_name +import os -if compat_os_name == 'nt': - # On older python versions, proxies are extracted from Windows registry erroneously. [1] +if os.name == 'nt': + # On older Python versions, proxies are extracted from Windows registry erroneously. [1] # If the https proxy in the registry does not have a scheme, urllib will incorrectly add https:// to it. [2] # It is unlikely that the user has actually set it to be https, so we should be fine to safely downgrade - # it to http on these older python versions to avoid issues + # it to http on these older Python versions to avoid issues # This also applies for ftp proxy type, as ftp:// proxy scheme is not supported. # 1: https://github.com/python/cpython/issues/86793 # 2: https://github.com/python/cpython/blob/51f1ae5ceb0673316c4e4b0175384e892e33cc6e/Lib/urllib/request.py#L2683-L2698 @@ -37,4 +37,4 @@ if compat_os_name == 'nt': def getproxies(): return getproxies_environment() or getproxies_registry_patched() -del compat_os_name +del os diff --git a/plugins/youtube_download/yt_dlp/cookies.py b/plugins/youtube_download/yt_dlp/cookies.py index a71fbc28..fad323c9 100644 --- a/plugins/youtube_download/yt_dlp/cookies.py +++ b/plugins/youtube_download/yt_dlp/cookies.py @@ -1,6 +1,10 @@ import base64 import collections import contextlib +import datetime as dt +import functools +import glob +import hashlib import http.cookiejar import http.cookies import io @@ -14,16 +18,13 @@ import sys import tempfile import time import urllib.request -from datetime import datetime, timedelta, timezone from enum import Enum, auto -from hashlib import pbkdf2_hmac from .aes import ( aes_cbc_decrypt_bytes, aes_gcm_decrypt_and_verify_bytes, unpad_pkcs7, ) -from .compat import functools from .dependencies import ( _SECRETSTORAGE_UNAVAILABLE_REASON, secretstorage, @@ -31,6 +32,8 @@ from .dependencies import ( ) from .minicurses import MultilinePrinter, QuietMultilinePrinter from .utils import ( + DownloadError, + YoutubeDLError, Popen, error_to_str, expand_path, @@ -43,7 +46,7 @@ from .utils import ( from .utils._utils import _YDLLogger from .utils.networking import normalize_url -CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'} +CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi', 'whale'} SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'} @@ -83,24 +86,31 @@ def _create_progress_bar(logger): return printer +class CookieLoadError(YoutubeDLError): + pass + + def load_cookies(cookie_file, browser_specification, ydl): - cookie_jars = [] - if browser_specification is not None: - browser_name, profile, keyring, container = _parse_browser_specification(*browser_specification) - cookie_jars.append( - extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl), keyring=keyring, container=container)) + try: + cookie_jars = [] + if browser_specification is not None: + browser_name, profile, keyring, container = _parse_browser_specification(*browser_specification) + cookie_jars.append( + extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl), keyring=keyring, container=container)) - if cookie_file is not None: - is_filename = is_path_like(cookie_file) - if is_filename: - cookie_file = expand_path(cookie_file) + if cookie_file is not None: + is_filename = is_path_like(cookie_file) + if is_filename: + cookie_file = expand_path(cookie_file) - jar = YoutubeDLCookieJar(cookie_file) - if not is_filename or os.access(cookie_file, os.R_OK): - jar.load() - cookie_jars.append(jar) + jar = YoutubeDLCookieJar(cookie_file) + if not is_filename or os.access(cookie_file, os.R_OK): + jar.load() + cookie_jars.append(jar) - return _merge_cookie_jars(cookie_jars) + return _merge_cookie_jars(cookie_jars) + except Exception: + raise CookieLoadError('failed to load cookies') def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(), *, keyring=None, container=None): @@ -118,17 +128,18 @@ def _extract_firefox_cookies(profile, container, logger): logger.info('Extracting cookies from firefox') if not sqlite3: logger.warning('Cannot extract cookies from firefox without sqlite3 support. ' - 'Please use a python interpreter compiled with sqlite3 support') + 'Please use a Python interpreter compiled with sqlite3 support') return YoutubeDLCookieJar() if profile is None: - search_root = _firefox_browser_dir() + search_roots = list(_firefox_browser_dirs()) elif _is_path(profile): - search_root = profile + search_roots = [profile] else: - search_root = os.path.join(_firefox_browser_dir(), profile) + search_roots = [os.path.join(path, profile) for path in _firefox_browser_dirs()] + search_root = ', '.join(map(repr, search_roots)) - cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite', logger) + cookie_database_path = _newest(_firefox_cookie_dbs(search_roots)) if cookie_database_path is None: raise FileNotFoundError(f'could not find firefox cookies database in {search_root}') logger.debug(f'Extracting cookies from: "{cookie_database_path}"') @@ -142,7 +153,7 @@ def _extract_firefox_cookies(profile, container, logger): identities = json.load(containers).get('identities', []) container_id = next((context.get('userContextId') for context in identities if container in ( context.get('name'), - try_call(lambda: re.fullmatch(r'userContext([^\.]+)\.label', context['l10nID']).group()) + try_call(lambda: re.fullmatch(r'userContext([^\.]+)\.label', context['l10nID']).group()), )), None) if not isinstance(container_id, int): raise ValueError(f'could not find firefox container "{container}" in containers.json') @@ -182,12 +193,28 @@ def _extract_firefox_cookies(profile, container, logger): cursor.connection.close() -def _firefox_browser_dir(): +def _firefox_browser_dirs(): if sys.platform in ('cygwin', 'win32'): - return os.path.expandvars(R'%APPDATA%\Mozilla\Firefox\Profiles') + yield from map(os.path.expandvars, ( + R'%APPDATA%\Mozilla\Firefox\Profiles', + R'%LOCALAPPDATA%\Packages\Mozilla.Firefox_n80bbvh6b1yt2\LocalCache\Roaming\Mozilla\Firefox\Profiles', + )) + elif sys.platform == 'darwin': - return os.path.expanduser('~/Library/Application Support/Firefox') - return os.path.expanduser('~/.mozilla/firefox') + yield os.path.expanduser('~/Library/Application Support/Firefox/Profiles') + + else: + yield from map(os.path.expanduser, ( + '~/.mozilla/firefox', + '~/snap/firefox/common/.mozilla/firefox', + '~/.var/app/org.mozilla.firefox/.mozilla/firefox', + )) + + +def _firefox_cookie_dbs(roots): + for root in map(os.path.abspath, roots): + for pattern in ('', '*/', 'Profiles/*/'): + yield from glob.iglob(os.path.join(root, pattern, 'cookies.sqlite')) def _get_chromium_based_browser_settings(browser_name): @@ -202,6 +229,7 @@ def _get_chromium_based_browser_settings(browser_name): 'edge': os.path.join(appdata_local, R'Microsoft\Edge\User Data'), 'opera': os.path.join(appdata_roaming, R'Opera Software\Opera Stable'), 'vivaldi': os.path.join(appdata_local, R'Vivaldi\User Data'), + 'whale': os.path.join(appdata_local, R'Naver\Naver Whale\User Data'), }[browser_name] elif sys.platform == 'darwin': @@ -213,6 +241,7 @@ def _get_chromium_based_browser_settings(browser_name): 'edge': os.path.join(appdata, 'Microsoft Edge'), 'opera': os.path.join(appdata, 'com.operasoftware.Opera'), 'vivaldi': os.path.join(appdata, 'Vivaldi'), + 'whale': os.path.join(appdata, 'Naver/Whale'), }[browser_name] else: @@ -224,6 +253,7 @@ def _get_chromium_based_browser_settings(browser_name): 'edge': os.path.join(config, 'microsoft-edge'), 'opera': os.path.join(config, 'opera'), 'vivaldi': os.path.join(config, 'vivaldi'), + 'whale': os.path.join(config, 'naver-whale'), }[browser_name] # Linux keyring names can be determined by snooping on dbus while opening the browser in KDE: @@ -235,6 +265,7 @@ def _get_chromium_based_browser_settings(browser_name): 'edge': 'Microsoft Edge' if sys.platform == 'darwin' else 'Chromium', 'opera': 'Opera' if sys.platform == 'darwin' else 'Chromium', 'vivaldi': 'Vivaldi' if sys.platform == 'darwin' else 'Chrome', + 'whale': 'Whale', }[browser_name] browsers_without_profiles = {'opera'} @@ -242,7 +273,7 @@ def _get_chromium_based_browser_settings(browser_name): return { 'browser_dir': browser_dir, 'keyring_name': keyring_name, - 'supports_profiles': browser_name not in browsers_without_profiles + 'supports_profiles': browser_name not in browsers_without_profiles, } @@ -251,7 +282,7 @@ def _extract_chrome_cookies(browser_name, profile, keyring, logger): if not sqlite3: logger.warning(f'Cannot extract cookies from {browser_name} without sqlite3 support. ' - 'Please use a python interpreter compiled with sqlite3 support') + 'Please use a Python interpreter compiled with sqlite3 support') return YoutubeDLCookieJar() config = _get_chromium_based_browser_settings(browser_name) @@ -268,17 +299,23 @@ def _extract_chrome_cookies(browser_name, profile, keyring, logger): logger.error(f'{browser_name} does not support profiles') search_root = config['browser_dir'] - cookie_database_path = _find_most_recently_used_file(search_root, 'Cookies', logger) + cookie_database_path = _newest(_find_files(search_root, 'Cookies', logger)) if cookie_database_path is None: raise FileNotFoundError(f'could not find {browser_name} cookies database in "{search_root}"') logger.debug(f'Extracting cookies from: "{cookie_database_path}"') - decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger, keyring=keyring) - with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir: cursor = None try: cursor = _open_database_copy(cookie_database_path, tmpdir) + + # meta_version is necessary to determine if we need to trim the hash prefix from the cookies + # Ref: https://chromium.googlesource.com/chromium/src/+/b02dcebd7cafab92770734dc2bc317bd07f1d891/net/extras/sqlite/sqlite_persistent_cookie_store.cc#223 + meta_version = int(cursor.execute('SELECT value FROM meta WHERE key = "version"').fetchone()[0]) + decryptor = get_cookie_decryptor( + config['browser_dir'], config['keyring_name'], logger, + keyring=keyring, meta_version=meta_version) + cursor.connection.text_factory = bytes column_names = _get_column_names(cursor, 'cookies') secure_column = 'is_secure' if 'is_secure' in column_names else 'secure' @@ -307,6 +344,12 @@ def _extract_chrome_cookies(browser_name, profile, keyring, logger): counts['unencrypted'] = unencrypted_cookies logger.debug(f'cookie version breakdown: {counts}') return jar + except PermissionError as error: + if os.name == 'nt' and error.errno == 13: + message = 'Could not copy Chrome cookie database. See https://github.com/yt-dlp/yt-dlp/issues/7271 for more info' + logger.error(message) + raise DownloadError(message) # force exit + raise finally: if cursor is not None: cursor.connection.close() @@ -324,6 +367,11 @@ def _process_chrome_cookie(decryptor, host_key, name, value, encrypted_value, pa if value is None: return is_encrypted, None + # In chrome, session cookies have expires_utc set to 0 + # In our cookie-store, cookies that do not expire should have expires set to None + if not expires_utc: + expires_utc = None + return is_encrypted, http.cookiejar.Cookie( version=0, name=name, value=value, port=None, port_specified=False, domain=host_key, domain_specified=bool(host_key), domain_initial_dot=host_key.startswith('.'), @@ -365,22 +413,23 @@ class ChromeCookieDecryptor: raise NotImplementedError('Must be implemented by sub classes') -def get_cookie_decryptor(browser_root, browser_keyring_name, logger, *, keyring=None): +def get_cookie_decryptor(browser_root, browser_keyring_name, logger, *, keyring=None, meta_version=None): if sys.platform == 'darwin': - return MacChromeCookieDecryptor(browser_keyring_name, logger) + return MacChromeCookieDecryptor(browser_keyring_name, logger, meta_version=meta_version) elif sys.platform in ('win32', 'cygwin'): - return WindowsChromeCookieDecryptor(browser_root, logger) - return LinuxChromeCookieDecryptor(browser_keyring_name, logger, keyring=keyring) + return WindowsChromeCookieDecryptor(browser_root, logger, meta_version=meta_version) + return LinuxChromeCookieDecryptor(browser_keyring_name, logger, keyring=keyring, meta_version=meta_version) class LinuxChromeCookieDecryptor(ChromeCookieDecryptor): - def __init__(self, browser_keyring_name, logger, *, keyring=None): + def __init__(self, browser_keyring_name, logger, *, keyring=None, meta_version=None): self._logger = logger self._v10_key = self.derive_key(b'peanuts') self._empty_key = self.derive_key(b'') self._cookie_counts = {'v10': 0, 'v11': 0, 'other': 0} self._browser_keyring_name = browser_keyring_name self._keyring = keyring + self._meta_version = meta_version or 0 @functools.cached_property def _v11_key(self): @@ -409,14 +458,18 @@ class LinuxChromeCookieDecryptor(ChromeCookieDecryptor): if version == b'v10': self._cookie_counts['v10'] += 1 - return _decrypt_aes_cbc_multi(ciphertext, (self._v10_key, self._empty_key), self._logger) + return _decrypt_aes_cbc_multi( + ciphertext, (self._v10_key, self._empty_key), self._logger, + hash_prefix=self._meta_version >= 24) elif version == b'v11': self._cookie_counts['v11'] += 1 if self._v11_key is None: self._logger.warning('cannot decrypt v11 cookies: no key found', only_once=True) return None - return _decrypt_aes_cbc_multi(ciphertext, (self._v11_key, self._empty_key), self._logger) + return _decrypt_aes_cbc_multi( + ciphertext, (self._v11_key, self._empty_key), self._logger, + hash_prefix=self._meta_version >= 24) else: self._logger.warning(f'unknown cookie version: "{version}"', only_once=True) @@ -425,11 +478,12 @@ class LinuxChromeCookieDecryptor(ChromeCookieDecryptor): class MacChromeCookieDecryptor(ChromeCookieDecryptor): - def __init__(self, browser_keyring_name, logger): + def __init__(self, browser_keyring_name, logger, meta_version=None): self._logger = logger password = _get_mac_keyring_password(browser_keyring_name, logger) self._v10_key = None if password is None else self.derive_key(password) self._cookie_counts = {'v10': 0, 'other': 0} + self._meta_version = meta_version or 0 @staticmethod def derive_key(password): @@ -447,7 +501,8 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor): self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True) return None - return _decrypt_aes_cbc_multi(ciphertext, (self._v10_key,), self._logger) + return _decrypt_aes_cbc_multi( + ciphertext, (self._v10_key,), self._logger, hash_prefix=self._meta_version >= 24) else: self._cookie_counts['other'] += 1 @@ -457,10 +512,11 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor): class WindowsChromeCookieDecryptor(ChromeCookieDecryptor): - def __init__(self, browser_root, logger): + def __init__(self, browser_root, logger, meta_version=None): self._logger = logger self._v10_key = _get_windows_v10_key(browser_root, logger) self._cookie_counts = {'v10': 0, 'other': 0} + self._meta_version = meta_version or 0 def decrypt(self, encrypted_value): version = encrypted_value[:3] @@ -484,7 +540,9 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor): ciphertext = raw_ciphertext[nonce_length:-authentication_tag_length] authentication_tag = raw_ciphertext[-authentication_tag_length:] - return _decrypt_aes_gcm(ciphertext, self._v10_key, nonce, authentication_tag, self._logger) + return _decrypt_aes_gcm( + ciphertext, self._v10_key, nonce, authentication_tag, self._logger, + hash_prefix=self._meta_version >= 24) else: self._cookie_counts['other'] += 1 @@ -575,7 +633,7 @@ class DataParser: def _mac_absolute_time_to_posix(timestamp): - return int((datetime(2001, 1, 1, 0, 0, tzinfo=timezone.utc) + timedelta(seconds=timestamp)).timestamp()) + return int((dt.datetime(2001, 1, 1, 0, 0, tzinfo=dt.timezone.utc) + dt.timedelta(seconds=timestamp)).timestamp()) def _parse_safari_cookies_header(data, logger): @@ -708,40 +766,38 @@ def _get_linux_desktop_environment(env, logger): xdg_current_desktop = env.get('XDG_CURRENT_DESKTOP', None) desktop_session = env.get('DESKTOP_SESSION', None) if xdg_current_desktop is not None: - xdg_current_desktop = xdg_current_desktop.split(':')[0].strip() - - if xdg_current_desktop == 'Unity': - if desktop_session is not None and 'gnome-fallback' in desktop_session: + for part in map(str.strip, xdg_current_desktop.split(':')): + if part == 'Unity': + if desktop_session is not None and 'gnome-fallback' in desktop_session: + return _LinuxDesktopEnvironment.GNOME + else: + return _LinuxDesktopEnvironment.UNITY + elif part == 'Deepin': + return _LinuxDesktopEnvironment.DEEPIN + elif part == 'GNOME': return _LinuxDesktopEnvironment.GNOME - else: - return _LinuxDesktopEnvironment.UNITY - elif xdg_current_desktop == 'Deepin': - return _LinuxDesktopEnvironment.DEEPIN - elif xdg_current_desktop == 'GNOME': - return _LinuxDesktopEnvironment.GNOME - elif xdg_current_desktop == 'X-Cinnamon': - return _LinuxDesktopEnvironment.CINNAMON - elif xdg_current_desktop == 'KDE': - kde_version = env.get('KDE_SESSION_VERSION', None) - if kde_version == '5': - return _LinuxDesktopEnvironment.KDE5 - elif kde_version == '6': - return _LinuxDesktopEnvironment.KDE6 - elif kde_version == '4': - return _LinuxDesktopEnvironment.KDE4 - else: - logger.info(f'unknown KDE version: "{kde_version}". Assuming KDE4') - return _LinuxDesktopEnvironment.KDE4 - elif xdg_current_desktop == 'Pantheon': - return _LinuxDesktopEnvironment.PANTHEON - elif xdg_current_desktop == 'XFCE': - return _LinuxDesktopEnvironment.XFCE - elif xdg_current_desktop == 'UKUI': - return _LinuxDesktopEnvironment.UKUI - elif xdg_current_desktop == 'LXQt': - return _LinuxDesktopEnvironment.LXQT - else: - logger.info(f'XDG_CURRENT_DESKTOP is set to an unknown value: "{xdg_current_desktop}"') + elif part == 'X-Cinnamon': + return _LinuxDesktopEnvironment.CINNAMON + elif part == 'KDE': + kde_version = env.get('KDE_SESSION_VERSION', None) + if kde_version == '5': + return _LinuxDesktopEnvironment.KDE5 + elif kde_version == '6': + return _LinuxDesktopEnvironment.KDE6 + elif kde_version == '4': + return _LinuxDesktopEnvironment.KDE4 + else: + logger.info(f'unknown KDE version: "{kde_version}". Assuming KDE4') + return _LinuxDesktopEnvironment.KDE4 + elif part == 'Pantheon': + return _LinuxDesktopEnvironment.PANTHEON + elif part == 'XFCE': + return _LinuxDesktopEnvironment.XFCE + elif part == 'UKUI': + return _LinuxDesktopEnvironment.UKUI + elif part == 'LXQt': + return _LinuxDesktopEnvironment.LXQT + logger.info(f'XDG_CURRENT_DESKTOP is set to an unknown value: "{xdg_current_desktop}"') elif desktop_session is not None: if desktop_session == 'deepin': @@ -794,7 +850,7 @@ def _choose_linux_keyring(logger): elif desktop_environment == _LinuxDesktopEnvironment.KDE6: linux_keyring = _LinuxKeyring.KWALLET6 elif desktop_environment in ( - _LinuxDesktopEnvironment.KDE3, _LinuxDesktopEnvironment.LXQT, _LinuxDesktopEnvironment.OTHER + _LinuxDesktopEnvironment.KDE3, _LinuxDesktopEnvironment.LXQT, _LinuxDesktopEnvironment.OTHER, ): linux_keyring = _LinuxKeyring.BASICTEXT else: @@ -829,7 +885,7 @@ def _get_kwallet_network_wallet(keyring, logger): 'dbus-send', '--session', '--print-reply=literal', f'--dest={service_name}', wallet_path, - 'org.kde.KWallet.networkWallet' + 'org.kde.KWallet.networkWallet', ], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) if returncode: @@ -859,7 +915,7 @@ def _get_kwallet_password(browser_keyring_name, keyring, logger): 'kwallet-query', '--read-password', f'{browser_keyring_name} Safe Storage', '--folder', f'{browser_keyring_name} Keys', - network_wallet + network_wallet, ], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) if returncode: @@ -899,9 +955,8 @@ def _get_gnome_keyring_password(browser_keyring_name, logger): for item in col.get_all_items(): if item.get_label() == f'{browser_keyring_name} Safe Storage': return item.get_secret() - else: - logger.error('failed to read from keyring') - return b'' + logger.error('failed to read from keyring') + return b'' def _get_linux_keyring_password(browser_keyring_name, keyring, logger): @@ -947,7 +1002,7 @@ def _get_windows_v10_key(browser_root, logger): References: - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_win.cc """ - path = _find_most_recently_used_file(browser_root, 'Local State', logger) + path = _newest(_find_files(browser_root, 'Local State', logger)) if path is None: logger.error('could not find local state file') return None @@ -970,13 +1025,15 @@ def _get_windows_v10_key(browser_root, logger): def pbkdf2_sha1(password, salt, iterations, key_length): - return pbkdf2_hmac('sha1', password, salt, iterations, key_length) + return hashlib.pbkdf2_hmac('sha1', password, salt, iterations, key_length) -def _decrypt_aes_cbc_multi(ciphertext, keys, logger, initialization_vector=b' ' * 16): +def _decrypt_aes_cbc_multi(ciphertext, keys, logger, initialization_vector=b' ' * 16, hash_prefix=False): for key in keys: plaintext = unpad_pkcs7(aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector)) try: + if hash_prefix: + return plaintext[32:].decode() return plaintext.decode() except UnicodeDecodeError: pass @@ -984,7 +1041,7 @@ def _decrypt_aes_cbc_multi(ciphertext, keys, logger, initialization_vector=b' ' return None -def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger): +def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger, hash_prefix=False): try: plaintext = aes_gcm_decrypt_and_verify_bytes(ciphertext, key, authentication_tag, nonce) except ValueError: @@ -992,6 +1049,8 @@ def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger): return None try: + if hash_prefix: + return plaintext[32:].decode() return plaintext.decode() except UnicodeDecodeError: logger.warning('failed to decrypt cookie (AES-GCM) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True) @@ -1021,11 +1080,12 @@ def _decrypt_windows_dpapi(ciphertext, logger): None, # pvReserved: must be NULL None, # pPromptStruct: information about prompts to display 0, # dwFlags - ctypes.byref(blob_out) # pDataOut + ctypes.byref(blob_out), # pDataOut ) if not ret: - logger.warning('failed to decrypt with DPAPI', only_once=True) - return None + message = 'Failed to decrypt with DPAPI. See https://github.com/yt-dlp/yt-dlp/issues/10927 for more info' + logger.error(message) + raise DownloadError(message) # force exit result = ctypes.string_at(blob_out.pbData, blob_out.cbData) ctypes.windll.kernel32.LocalFree(blob_out.pbData) @@ -1049,17 +1109,20 @@ def _get_column_names(cursor, table_name): return [row[1].decode() for row in table_info] -def _find_most_recently_used_file(root, filename, logger): +def _newest(files): + return max(files, key=lambda path: os.lstat(path).st_mtime, default=None) + + +def _find_files(root, filename, logger): # if there are multiple browser profiles, take the most recently used one - i, paths = 0, [] + i = 0 with _create_progress_bar(logger) as progress_bar: - for curr_root, dirs, files in os.walk(root): + for curr_root, _, files in os.walk(root): for file in files: i += 1 progress_bar.print(f'Searching for "{filename}": {i: 6d} files searched') if file == filename: - paths.append(os.path.join(curr_root, file)) - return None if not paths else max(paths, key=lambda path: os.lstat(path).st_mtime) + yield os.path.join(curr_root, file) def _merge_cookie_jars(jars): @@ -1073,7 +1136,7 @@ def _merge_cookie_jars(jars): def _is_path(value): - return os.path.sep in value + return any(sep in value for sep in (os.path.sep, os.path.altsep) if sep) def _parse_browser_specification(browser_name, profile=None, keyring=None, container=None): @@ -1094,24 +1157,24 @@ class LenientSimpleCookie(http.cookies.SimpleCookie): _LEGAL_VALUE_CHARS = _LEGAL_KEY_CHARS + re.escape('(),/<=>?@[]{}') _RESERVED = { - "expires", - "path", - "comment", - "domain", - "max-age", - "secure", - "httponly", - "version", - "samesite", + 'expires', + 'path', + 'comment', + 'domain', + 'max-age', + 'secure', + 'httponly', + 'version', + 'samesite', } - _FLAGS = {"secure", "httponly"} + _FLAGS = {'secure', 'httponly'} # Added 'bad' group to catch the remaining value - _COOKIE_PATTERN = re.compile(r""" + _COOKIE_PATTERN = re.compile(r''' \s* # Optional whitespace at start of cookie (?P # Start of group 'key' - [""" + _LEGAL_KEY_CHARS + r"""]+?# Any word of at least one letter + [''' + _LEGAL_KEY_CHARS + r''']+?# Any word of at least one letter ) # End of group 'key' ( # Optional group: there may not be a value. \s*=\s* # Equal Sign @@ -1121,7 +1184,7 @@ class LenientSimpleCookie(http.cookies.SimpleCookie): | # or \w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr | # or - [""" + _LEGAL_VALUE_CHARS + r"""]* # Any word or empty string + [''' + _LEGAL_VALUE_CHARS + r''']* # Any word or empty string ) # End of group 'val' | # or (?P(?:\\;|[^;])*?) # 'bad' group fallback for invalid values @@ -1129,7 +1192,7 @@ class LenientSimpleCookie(http.cookies.SimpleCookie): )? # End of optional value group \s* # Any number of spaces. (\s+|;|$) # Ending either at space, semicolon, or EOS. - """, re.ASCII | re.VERBOSE) + ''', re.ASCII | re.VERBOSE) def load(self, data): # Workaround for https://github.com/yt-dlp/yt-dlp/issues/4776 @@ -1216,8 +1279,8 @@ class YoutubeDLCookieJar(http.cookiejar.MozillaCookieJar): def _really_save(self, f, ignore_discard, ignore_expires): now = time.time() for cookie in self: - if (not ignore_discard and cookie.discard - or not ignore_expires and cookie.is_expired(now)): + if ((not ignore_discard and cookie.discard) + or (not ignore_expires and cookie.is_expired(now))): continue name, value = cookie.name, cookie.value if value is None: @@ -1225,14 +1288,14 @@ class YoutubeDLCookieJar(http.cookiejar.MozillaCookieJar): # with no name, whereas http.cookiejar regards it as a # cookie with no value. name, value = '', name - f.write('%s\n' % '\t'.join(( + f.write('{}\n'.format('\t'.join(( cookie.domain, self._true_or_false(cookie.domain.startswith('.')), cookie.path, self._true_or_false(cookie.secure), str_or_none(cookie.expires, default=''), - name, value - ))) + name, value, + )))) def save(self, filename=None, ignore_discard=True, ignore_expires=True): """ @@ -1271,10 +1334,10 @@ class YoutubeDLCookieJar(http.cookiejar.MozillaCookieJar): return line cookie_list = line.split('\t') if len(cookie_list) != self._ENTRY_LEN: - raise http.cookiejar.LoadError('invalid length %d' % len(cookie_list)) + raise http.cookiejar.LoadError(f'invalid length {len(cookie_list)}') cookie = self._CookieFileEntry(*cookie_list) if cookie.expires_at and not cookie.expires_at.isdigit(): - raise http.cookiejar.LoadError('invalid expires at %s' % cookie.expires_at) + raise http.cookiejar.LoadError(f'invalid expires at {cookie.expires_at}') return line cf = io.StringIO() diff --git a/plugins/youtube_download/yt_dlp/dependencies/Cryptodome.py b/plugins/youtube_download/yt_dlp/dependencies/Cryptodome.py index 2cfa4c95..0e4404d4 100644 --- a/plugins/youtube_download/yt_dlp/dependencies/Cryptodome.py +++ b/plugins/youtube_download/yt_dlp/dependencies/Cryptodome.py @@ -24,7 +24,7 @@ try: from Crypto.Cipher import AES, PKCS1_OAEP, Blowfish, PKCS1_v1_5 # noqa: F401 from Crypto.Hash import CMAC, SHA1 # noqa: F401 from Crypto.PublicKey import RSA # noqa: F401 -except ImportError: +except (ImportError, OSError): __version__ = f'broken {__version__}'.strip() diff --git a/plugins/youtube_download/yt_dlp/dependencies/__init__.py b/plugins/youtube_download/yt_dlp/dependencies/__init__.py index 6e7d29c5..0d58da2b 100644 --- a/plugins/youtube_download/yt_dlp/dependencies/__init__.py +++ b/plugins/youtube_download/yt_dlp/dependencies/__init__.py @@ -43,19 +43,28 @@ except Exception as _err: try: import sqlite3 + # We need to get the underlying `sqlite` version, see https://github.com/yt-dlp/yt-dlp/issues/8152 + sqlite3._yt_dlp__version = sqlite3.sqlite_version except ImportError: - # although sqlite3 is part of the standard library, it is possible to compile python without + # although sqlite3 is part of the standard library, it is possible to compile Python without # sqlite support. See: https://github.com/yt-dlp/yt-dlp/issues/544 sqlite3 = None try: import websockets -except (ImportError, SyntaxError): - # websockets 3.10 on python 3.6 causes SyntaxError - # See https://github.com/yt-dlp/yt-dlp/issues/2633 +except ImportError: websockets = None +try: + import urllib3 +except ImportError: + urllib3 = None + +try: + import requests +except ImportError: + requests = None try: import xattr # xattr or pyxattr @@ -65,6 +74,10 @@ else: if hasattr(xattr, 'set'): # pyxattr xattr._yt_dlp__identifier = 'pyxattr' +try: + import curl_cffi +except ImportError: + curl_cffi = None from . import Cryptodome diff --git a/plugins/youtube_download/yt_dlp/downloader/__init__.py b/plugins/youtube_download/yt_dlp/downloader/__init__.py index 51a9f28f..9c34bd28 100644 --- a/plugins/youtube_download/yt_dlp/downloader/__init__.py +++ b/plugins/youtube_download/yt_dlp/downloader/__init__.py @@ -30,11 +30,12 @@ from .hls import HlsFD from .http import HttpFD from .ism import IsmFD from .mhtml import MhtmlFD -from .niconico import NiconicoDmcFD, NiconicoLiveFD +from .niconico import NiconicoLiveFD from .rtmp import RtmpFD from .rtsp import RtspFD from .websocket import WebSocketFragmentFD from .youtube_live_chat import YoutubeLiveChatFD +from .bunnycdn import BunnyCdnFD PROTOCOL_MAP = { 'rtmp': RtmpFD, @@ -49,12 +50,12 @@ PROTOCOL_MAP = { 'http_dash_segments_generator': DashSegmentsFD, 'ism': IsmFD, 'mhtml': MhtmlFD, - 'niconico_dmc': NiconicoDmcFD, 'niconico_live': NiconicoLiveFD, 'fc2_live': FC2LiveFD, 'websocket_frag': WebSocketFragmentFD, 'youtube_live_chat': YoutubeLiveChatFD, 'youtube_live_chat_replay': YoutubeLiveChatFD, + 'bunnycdn': BunnyCdnFD, } @@ -65,7 +66,6 @@ def shorten_protocol_name(proto, simplify=False): 'rtmp_ffmpeg': 'rtmpF', 'http_dash_segments': 'dash', 'http_dash_segments_generator': 'dashG', - 'niconico_dmc': 'dmc', 'websocket_frag': 'WSfrag', } if simplify: diff --git a/plugins/youtube_download/yt_dlp/downloader/bunnycdn.py b/plugins/youtube_download/yt_dlp/downloader/bunnycdn.py new file mode 100644 index 00000000..e787f698 --- /dev/null +++ b/plugins/youtube_download/yt_dlp/downloader/bunnycdn.py @@ -0,0 +1,50 @@ +import hashlib +import random +import threading + +from .common import FileDownloader +from . import HlsFD +from ..networking import Request +from ..networking.exceptions import network_exceptions + + +class BunnyCdnFD(FileDownloader): + """ + Downloads from BunnyCDN with required pings + Note, this is not a part of public API, and will be removed without notice. + DO NOT USE + """ + + def real_download(self, filename, info_dict): + self.to_screen(f'[{self.FD_NAME}] Downloading from BunnyCDN') + + fd = HlsFD(self.ydl, self.params) + + stop_event = threading.Event() + ping_thread = threading.Thread(target=self.ping_thread, args=(stop_event,), kwargs=info_dict['_bunnycdn_ping_data']) + ping_thread.start() + + try: + return fd.real_download(filename, info_dict) + finally: + stop_event.set() + + def ping_thread(self, stop_event, url, headers, secret, context_id): + # Site sends ping every 4 seconds, but this throttles the download. Pinging every 2 seconds seems to work. + ping_interval = 2 + # Hard coded resolution as it doesn't seem to matter + res = 1080 + paused = 'false' + current_time = 0 + + while not stop_event.wait(ping_interval): + current_time += ping_interval + + time = current_time + round(random.random(), 6) + md5_hash = hashlib.md5(f'{secret}_{context_id}_{time}_{paused}_{res}'.encode()).hexdigest() + ping_url = f'{url}?hash={md5_hash}&time={time}&paused={paused}&resolution={res}' + + try: + self.ydl.urlopen(Request(ping_url, headers=headers)).read() + except network_exceptions as e: + self.to_screen(f'[{self.FD_NAME}] Ping failed: {e}') diff --git a/plugins/youtube_download/yt_dlp/downloader/common.py b/plugins/youtube_download/yt_dlp/downloader/common.py index b71d7ee8..bb9303f8 100644 --- a/plugins/youtube_download/yt_dlp/downloader/common.py +++ b/plugins/youtube_download/yt_dlp/downloader/common.py @@ -4,6 +4,7 @@ import functools import os import random import re +import threading import time from ..minicurses import ( @@ -19,9 +20,7 @@ from ..utils import ( Namespace, RetryManager, classproperty, - decodeArgument, deprecation_warning, - encodeFilename, format_bytes, join_nonempty, parse_bytes, @@ -32,6 +31,7 @@ from ..utils import ( timetuple_from_msec, try_call, ) +from ..utils._utils import _ProgressState class FileDownloader: @@ -63,6 +63,7 @@ class FileDownloader: min_filesize: Skip files smaller than this size max_filesize: Skip files larger than this size xattr_set_filesize: Set ytdl.filesize user xattribute with expected size. + progress_delta: The minimum time between progress output, in seconds external_downloader_args: A dictionary of downloader keys (in lower case) and a list of additional command-line arguments for the executable. Use 'default' as the name for arguments to be @@ -88,6 +89,9 @@ class FileDownloader: self.params = params self._prepare_multiline_status() self.add_progress_hook(self.report_progress) + if self.params.get('progress_delta'): + self._progress_delta_lock = threading.Lock() + self._progress_delta_time = time.monotonic() def _set_ydl(self, ydl): self.ydl = ydl @@ -214,7 +218,7 @@ class FileDownloader: def temp_name(self, filename): """Returns a temporary filename for the given filename.""" if self.params.get('nopart', False) or filename == '-' or \ - (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))): + (os.path.exists(filename) and not os.path.isfile(filename)): return filename return filename + '.part' @@ -268,7 +272,7 @@ class FileDownloader: """Try to set the last-modified time of the given file.""" if last_modified_hdr is None: return - if not os.path.isfile(encodeFilename(filename)): + if not os.path.isfile(filename): return timestr = last_modified_hdr if timestr is None: @@ -330,7 +334,7 @@ class FileDownloader: progress_dict), s.get('progress_idx') or 0) self.to_console_title(self.ydl.evaluate_outtmpl( progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s', - progress_dict)) + progress_dict), _ProgressState.from_dict(s), s.get('_percent')) def _format_progress(self, *args, **kwargs): return self.ydl._format_text( @@ -354,6 +358,7 @@ class FileDownloader: '_speed_str': self.format_speed(speed).strip(), '_total_bytes_str': _format_bytes('total_bytes'), '_elapsed_str': self.format_seconds(s.get('elapsed')), + '_percent': 100.0, '_percent_str': self.format_percent(100), }) self._report_progress_status(s, join_nonempty( @@ -366,13 +371,21 @@ class FileDownloader: if s['status'] != 'downloading': return + if update_delta := self.params.get('progress_delta'): + with self._progress_delta_lock: + if time.monotonic() < self._progress_delta_time: + return + self._progress_delta_time += update_delta + + progress = try_call( + lambda: 100 * s['downloaded_bytes'] / s['total_bytes'], + lambda: 100 * s['downloaded_bytes'] / s['total_bytes_estimate'], + lambda: s['downloaded_bytes'] == 0 and 0) s.update({ '_eta_str': self.format_eta(s.get('eta')).strip(), '_speed_str': self.format_speed(s.get('speed')), - '_percent_str': self.format_percent(try_call( - lambda: 100 * s['downloaded_bytes'] / s['total_bytes'], - lambda: 100 * s['downloaded_bytes'] / s['total_bytes_estimate'], - lambda: s['downloaded_bytes'] == 0 and 0)), + '_percent': progress, + '_percent_str': self.format_percent(progress), '_total_bytes_str': _format_bytes('total_bytes'), '_total_bytes_estimate_str': _format_bytes('total_bytes_estimate'), '_downloaded_bytes_str': _format_bytes('downloaded_bytes'), @@ -393,7 +406,7 @@ class FileDownloader: def report_resuming_byte(self, resume_len): """Report attempt to resume at given byte.""" - self.to_screen('[download] Resuming download at byte %s' % resume_len) + self.to_screen(f'[download] Resuming download at byte {resume_len}') def report_retry(self, err, count, retries, frag_index=NO_DEFAULT, fatal=True): """Report retry""" @@ -421,13 +434,13 @@ class FileDownloader: """ nooverwrites_and_exists = ( not self.params.get('overwrites', True) - and os.path.exists(encodeFilename(filename)) + and os.path.exists(filename) ) if not hasattr(filename, 'write'): continuedl_and_exists = ( self.params.get('continuedl', True) - and os.path.isfile(encodeFilename(filename)) + and os.path.isfile(filename) and not self.params.get('nopart', False) ) @@ -437,7 +450,7 @@ class FileDownloader: self._hook_progress({ 'filename': filename, 'status': 'finished', - 'total_bytes': os.path.getsize(encodeFilename(filename)), + 'total_bytes': os.path.getsize(filename), }, info_dict) self._finish_multiline_status() return True, False @@ -478,9 +491,7 @@ class FileDownloader: if not self.params.get('verbose', False): return - str_args = [decodeArgument(a) for a in args] - if exe is None: - exe = os.path.basename(str_args[0]) + exe = os.path.basename(args[0]) - self.write_debug(f'{exe} command line: {shell_quote(str_args)}') + self.write_debug(f'{exe} command line: {shell_quote(args)}') diff --git a/plugins/youtube_download/yt_dlp/downloader/dash.py b/plugins/youtube_download/yt_dlp/downloader/dash.py index 4328d739..afc79b6c 100644 --- a/plugins/youtube_download/yt_dlp/downloader/dash.py +++ b/plugins/youtube_download/yt_dlp/downloader/dash.py @@ -15,12 +15,15 @@ class DashSegmentsFD(FragmentFD): FD_NAME = 'dashsegments' def real_download(self, filename, info_dict): - if info_dict.get('is_live') and set(info_dict['protocol'].split('+')) != {'http_dash_segments_generator'}: - self.report_error('Live DASH videos are not supported') + if 'http_dash_segments_generator' in info_dict['protocol'].split('+'): + real_downloader = None # No external FD can support --live-from-start + else: + if info_dict.get('is_live'): + self.report_error('Live DASH videos are not supported') + real_downloader = get_suitable_downloader( + info_dict, self.params, None, protocol='dash_frag_urls', to_stdout=(filename == '-')) real_start = time.time() - real_downloader = get_suitable_downloader( - info_dict, self.params, None, protocol='dash_frag_urls', to_stdout=(filename == '-')) requested_formats = [{**info_dict, **fmt} for fmt in info_dict.get('requested_formats', [])] args = [] diff --git a/plugins/youtube_download/yt_dlp/downloader/external.py b/plugins/youtube_download/yt_dlp/downloader/external.py index 4ce8a3bf..ee73ac04 100644 --- a/plugins/youtube_download/yt_dlp/downloader/external.py +++ b/plugins/youtube_download/yt_dlp/downloader/external.py @@ -1,4 +1,5 @@ import enum +import functools import json import os import re @@ -9,7 +10,6 @@ import time import uuid from .fragment import FragmentFD -from ..compat import functools from ..networking import Request from ..postprocessor.ffmpeg import EXT_TO_OUT_FORMATS, FFmpegPostProcessor from ..utils import ( @@ -23,7 +23,6 @@ from ..utils import ( cli_valueless_option, determine_ext, encodeArgument, - encodeFilename, find_available_port, remove_end, traverse_obj, @@ -55,7 +54,7 @@ class ExternalFD(FragmentFD): # correct and expected termination thus all postprocessing # should take place retval = 0 - self.to_screen('[%s] Interrupted by user' % self.get_basename()) + self.to_screen(f'[{self.get_basename()}] Interrupted by user') finally: if self._cookies_tempfile: self.try_remove(self._cookies_tempfile) @@ -67,7 +66,7 @@ class ExternalFD(FragmentFD): 'elapsed': time.time() - started, } if filename != '-': - fsize = os.path.getsize(encodeFilename(tmpfilename)) + fsize = os.path.getsize(tmpfilename) self.try_rename(tmpfilename, filename) status.update({ 'downloaded_bytes': fsize, @@ -108,7 +107,7 @@ class ExternalFD(FragmentFD): return all(( not info_dict.get('to_stdout') or Features.TO_STDOUT in cls.SUPPORTED_FEATURES, '+' not in info_dict['protocol'] or Features.MULTIPLE_FORMATS in cls.SUPPORTED_FEATURES, - not traverse_obj(info_dict, ('hls_aes', ...), 'extra_param_to_segment_url'), + not traverse_obj(info_dict, ('hls_aes', ...), 'extra_param_to_segment_url', 'extra_param_to_key_url'), all(proto in cls.SUPPORTED_PROTOCOLS for proto in info_dict['protocol'].split('+')), )) @@ -172,7 +171,7 @@ class ExternalFD(FragmentFD): decrypt_fragment = self.decrypter(info_dict) dest, _ = self.sanitize_open(tmpfilename, 'wb') for frag_index, fragment in enumerate(info_dict['fragments']): - fragment_filename = '%s-Frag%d' % (tmpfilename, frag_index) + fragment_filename = f'{tmpfilename}-Frag{frag_index}' try: src, _ = self.sanitize_open(fragment_filename, 'rb') except OSError as err: @@ -184,9 +183,9 @@ class ExternalFD(FragmentFD): dest.write(decrypt_fragment(fragment, src.read())) src.close() if not self.params.get('keep_fragments', False): - self.try_remove(encodeFilename(fragment_filename)) + self.try_remove(fragment_filename) dest.close() - self.try_remove(encodeFilename('%s.frag.urls' % tmpfilename)) + self.try_remove(f'{tmpfilename}.frag.urls') return 0 def _call_process(self, cmd, info_dict): @@ -335,12 +334,12 @@ class Aria2cFD(ExternalFD): cmd += ['--auto-file-renaming=false'] if 'fragments' in info_dict: - cmd += ['--file-allocation=none', '--uri-selector=inorder'] - url_list_file = '%s.frag.urls' % tmpfilename + cmd += ['--uri-selector=inorder'] + url_list_file = f'{tmpfilename}.frag.urls' url_list = [] for frag_index, fragment in enumerate(info_dict['fragments']): - fragment_filename = '%s-Frag%d' % (os.path.basename(tmpfilename), frag_index) - url_list.append('%s\n\tout=%s' % (fragment['url'], self._aria2c_filename(fragment_filename))) + fragment_filename = f'{os.path.basename(tmpfilename)}-Frag{frag_index}' + url_list.append('{}\n\tout={}'.format(fragment['url'], self._aria2c_filename(fragment_filename))) stream, _ = self.sanitize_open(url_list_file, 'wb') stream.write('\n'.join(url_list).encode()) stream.close() @@ -357,7 +356,7 @@ class Aria2cFD(ExternalFD): 'id': sanitycheck, 'method': method, 'params': [f'token:{rpc_secret}', *params], - }).encode('utf-8') + }).encode() request = Request( f'http://localhost:{rpc_port}/jsonrpc', data=d, headers={ @@ -416,7 +415,7 @@ class Aria2cFD(ExternalFD): 'total_bytes_estimate': total, 'eta': (total - downloaded) / (speed or 1), 'fragment_index': min(frag_count, len(completed) + 1) if fragmented else None, - 'elapsed': time.time() - started + 'elapsed': time.time() - started, }) self._hook_progress(status, info_dict) @@ -458,8 +457,6 @@ class FFmpegFD(ExternalFD): @classmethod def available(cls, path=None): - # TODO: Fix path for ffmpeg - # Fixme: This may be wrong when --ffmpeg-location is used return FFmpegPostProcessor().available def on_process_started(self, proc, stdin): @@ -491,7 +488,7 @@ class FFmpegFD(ExternalFD): if not self.params.get('verbose'): args += ['-hide_banner'] - args += traverse_obj(info_dict, ('downloader_options', 'ffmpeg_args'), default=[]) + args += traverse_obj(info_dict, ('downloader_options', 'ffmpeg_args', ...)) # These exists only for compatibility. Extractors should use # info_dict['downloader_options']['ffmpeg_args'] instead @@ -508,13 +505,13 @@ class FFmpegFD(ExternalFD): env = None proxy = self.params.get('proxy') if proxy: - if not re.match(r'^[\da-zA-Z]+://', proxy): - proxy = 'http://%s' % proxy + if not re.match(r'[\da-zA-Z]+://', proxy): + proxy = f'http://{proxy}' if proxy.startswith('socks'): self.report_warning( - '%s does not support SOCKS proxies. Downloading is likely to fail. ' - 'Consider adding --hls-prefer-native to your command.' % self.get_basename()) + f'{self.get_basename()} does not support SOCKS proxies. Downloading is likely to fail. ' + 'Consider adding --hls-prefer-native to your command.') # Since December 2015 ffmpeg supports -http_proxy option (see # http://git.videolan.org/?p=ffmpeg.git;a=commit;h=b4eb1f29ebddd60c41a2eb39f5af701e38e0d3fd) @@ -559,7 +556,7 @@ class FFmpegFD(ExternalFD): selected_formats = info_dict.get('requested_formats') or [info_dict] for i, fmt in enumerate(selected_formats): - is_http = re.match(r'^https?://', fmt['url']) + is_http = re.match(r'https?://', fmt['url']) cookies = self.ydl.cookiejar.get_cookies_for_url(fmt['url']) if is_http else [] if cookies: args.extend(['-cookies', ''.join( @@ -575,7 +572,7 @@ class FFmpegFD(ExternalFD): if end_time: args += ['-t', str(end_time - start_time)] - args += self._configuration_args((f'_i{i + 1}', '_i')) + ['-i', fmt['url']] + args += [*self._configuration_args((f'_i{i + 1}', '_i')), '-i', fmt['url']] if not (start_time or end_time) or not self.params.get('force_keyframes_at_cuts'): args += ['-c', 'copy'] @@ -615,10 +612,12 @@ class FFmpegFD(ExternalFD): else: args += ['-f', EXT_TO_OUT_FORMATS.get(ext, ext)] + args += traverse_obj(info_dict, ('downloader_options', 'ffmpeg_args_out', ...)) + args += self._configuration_args(('_o1', '_o', '')) args = [encodeArgument(opt) for opt in args] - args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True)) + args.append(ffpp._ffmpeg_filename_argument(tmpfilename)) self._debug_cmd(args) piped = any(fmt['url'] in ('-', 'pipe:') for fmt in selected_formats) diff --git a/plugins/youtube_download/yt_dlp/downloader/f4m.py b/plugins/youtube_download/yt_dlp/downloader/f4m.py index 28cbba01..22d0ebd2 100644 --- a/plugins/youtube_download/yt_dlp/downloader/f4m.py +++ b/plugins/youtube_download/yt_dlp/downloader/f4m.py @@ -67,12 +67,12 @@ class FlvReader(io.BytesIO): self.read_bytes(3) quality_entry_count = self.read_unsigned_char() # QualityEntryCount - for i in range(quality_entry_count): + for _ in range(quality_entry_count): self.read_string() segment_run_count = self.read_unsigned_int() segments = [] - for i in range(segment_run_count): + for _ in range(segment_run_count): first_segment = self.read_unsigned_int() fragments_per_segment = self.read_unsigned_int() segments.append((first_segment, fragments_per_segment)) @@ -91,12 +91,12 @@ class FlvReader(io.BytesIO): quality_entry_count = self.read_unsigned_char() # QualitySegmentUrlModifiers - for i in range(quality_entry_count): + for _ in range(quality_entry_count): self.read_string() fragments_count = self.read_unsigned_int() fragments = [] - for i in range(fragments_count): + for _ in range(fragments_count): first = self.read_unsigned_int() first_ts = self.read_unsigned_long_long() duration = self.read_unsigned_int() @@ -135,11 +135,11 @@ class FlvReader(io.BytesIO): self.read_string() # MovieIdentifier server_count = self.read_unsigned_char() # ServerEntryTable - for i in range(server_count): + for _ in range(server_count): self.read_string() quality_count = self.read_unsigned_char() # QualityEntryTable - for i in range(quality_count): + for _ in range(quality_count): self.read_string() # DrmData self.read_string() @@ -148,14 +148,14 @@ class FlvReader(io.BytesIO): segments_count = self.read_unsigned_char() segments = [] - for i in range(segments_count): + for _ in range(segments_count): box_size, box_type, box_data = self.read_box_info() assert box_type == b'asrt' segment = FlvReader(box_data).read_asrt() segments.append(segment) fragments_run_count = self.read_unsigned_char() fragments = [] - for i in range(fragments_run_count): + for _ in range(fragments_run_count): box_size, box_type, box_data = self.read_box_info() assert box_type == b'afrt' fragments.append(FlvReader(box_data).read_afrt()) @@ -309,7 +309,7 @@ class F4mFD(FragmentFD): def real_download(self, filename, info_dict): man_url = info_dict['url'] requested_bitrate = info_dict.get('tbr') - self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME) + self.to_screen(f'[{self.FD_NAME}] Downloading f4m manifest') urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url)) man_url = urlh.url @@ -326,8 +326,8 @@ class F4mFD(FragmentFD): formats = sorted(formats, key=lambda f: f[0]) rate, media = formats[-1] else: - rate, media = list(filter( - lambda f: int(f[0]) == requested_bitrate, formats))[0] + rate, media = next(filter( + lambda f: int(f[0]) == requested_bitrate, formats)) # Prefer baseURL for relative URLs as per 11.2 of F4M 3.0 spec. man_base_url = get_base_url(doc) or man_url diff --git a/plugins/youtube_download/yt_dlp/downloader/fragment.py b/plugins/youtube_download/yt_dlp/downloader/fragment.py index b4b680da..98784e70 100644 --- a/plugins/youtube_download/yt_dlp/downloader/fragment.py +++ b/plugins/youtube_download/yt_dlp/downloader/fragment.py @@ -9,11 +9,11 @@ import time from .common import FileDownloader from .http import HttpFD from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7 -from ..compat import compat_os_name from ..networking import Request from ..networking.exceptions import HTTPError, IncompleteRead -from ..utils import DownloadError, RetryManager, encodeFilename, traverse_obj +from ..utils import DownloadError, RetryManager, traverse_obj from ..utils.networking import HTTPHeaderDict +from ..utils.progress import ProgressCalculator class HttpQuietDownloader(HttpFD): @@ -151,7 +151,7 @@ class FragmentFD(FileDownloader): if self.__do_ytdl_file(ctx): self._write_ytdl_file(ctx) if not self.params.get('keep_fragments', False): - self.try_remove(encodeFilename(ctx['fragment_filename_sanitized'])) + self.try_remove(ctx['fragment_filename_sanitized']) del ctx['fragment_filename_sanitized'] def _prepare_frag_download(self, ctx): @@ -187,7 +187,7 @@ class FragmentFD(FileDownloader): }) if self.__do_ytdl_file(ctx): - ytdl_file_exists = os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))) + ytdl_file_exists = os.path.isfile(self.ytdl_filename(ctx['filename'])) continuedl = self.params.get('continuedl', True) if continuedl and ytdl_file_exists: self._read_ytdl_file(ctx) @@ -198,7 +198,7 @@ class FragmentFD(FileDownloader): '.ytdl file is corrupt' if is_corrupt else 'Inconsistent state of incomplete fragment download') self.report_warning( - '%s. Restarting from the beginning ...' % message) + f'{message}. Restarting from the beginning ...') ctx['fragment_index'] = resume_len = 0 if 'ytdl_corrupt' in ctx: del ctx['ytdl_corrupt'] @@ -226,8 +226,7 @@ class FragmentFD(FileDownloader): resume_len = ctx['complete_frags_downloaded_bytes'] total_frags = ctx['total_frags'] ctx_id = ctx.get('ctx_id') - # This dict stores the download progress, it's updated by the progress - # hook + # Stores the download progress, updated by the progress hook state = { 'status': 'downloading', 'downloaded_bytes': resume_len, @@ -237,14 +236,8 @@ class FragmentFD(FileDownloader): 'tmpfilename': ctx['tmpfilename'], } - start = time.time() - ctx.update({ - 'started': start, - 'fragment_started': start, - # Amount of fragment's bytes downloaded by the time of the previous - # frag progress hook invocation - 'prev_frag_downloaded_bytes': 0, - }) + ctx['started'] = time.time() + progress = ProgressCalculator(resume_len) def frag_progress_hook(s): if s['status'] not in ('downloading', 'finished'): @@ -259,38 +252,35 @@ class FragmentFD(FileDownloader): state['max_progress'] = ctx.get('max_progress') state['progress_idx'] = ctx.get('progress_idx') - time_now = time.time() - state['elapsed'] = time_now - start + state['elapsed'] = progress.elapsed frag_total_bytes = s.get('total_bytes') or 0 s['fragment_info_dict'] = s.pop('info_dict', {}) + + # XXX: Fragment resume is not accounted for here if not ctx['live']: estimated_size = ( (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) / (state['fragment_index'] + 1) * total_frags) - state['total_bytes_estimate'] = estimated_size + progress.total = estimated_size + progress.update(s.get('downloaded_bytes')) + state['total_bytes_estimate'] = progress.total + else: + progress.update(s.get('downloaded_bytes')) if s['status'] == 'finished': state['fragment_index'] += 1 ctx['fragment_index'] = state['fragment_index'] - state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes'] - ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes'] - ctx['speed'] = state['speed'] = self.calc_speed( - ctx['fragment_started'], time_now, frag_total_bytes) - ctx['fragment_started'] = time.time() - ctx['prev_frag_downloaded_bytes'] = 0 - else: - frag_downloaded_bytes = s['downloaded_bytes'] - state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes'] - ctx['speed'] = state['speed'] = self.calc_speed( - ctx['fragment_started'], time_now, frag_downloaded_bytes - ctx.get('frag_resume_len', 0)) - if not ctx['live']: - state['eta'] = self.calc_eta(state['speed'], estimated_size - state['downloaded_bytes']) - ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes + progress.thread_reset() + + state['downloaded_bytes'] = ctx['complete_frags_downloaded_bytes'] = progress.downloaded + state['speed'] = ctx['speed'] = progress.speed.smooth + state['eta'] = progress.eta.smooth + self._hook_progress(state, info_dict) ctx['dl'].add_progress_hook(frag_progress_hook) - return start + return ctx['started'] def _finish_frag_download(self, ctx, info_dict): ctx['dest_stream'].close() @@ -375,10 +365,10 @@ class FragmentFD(FileDownloader): return decrypt_fragment def download_and_append_fragments_multiple(self, *args, **kwargs): - ''' + """ @params (ctx1, fragments1, info_dict1), (ctx2, fragments2, info_dict2), ... all args must be either tuple or list - ''' + """ interrupt_trigger = [True] max_progress = len(args) if max_progress == 1: @@ -399,7 +389,7 @@ class FragmentFD(FileDownloader): def __exit__(self, exc_type, exc_val, exc_tb): pass - if compat_os_name == 'nt': + if os.name == 'nt': def future_result(future): while True: try: @@ -433,7 +423,7 @@ class FragmentFD(FileDownloader): finally: tpe.shutdown(wait=True) if not interrupt_trigger[0] and not is_live: - raise KeyboardInterrupt() + raise KeyboardInterrupt # we expect the user wants to stop and DO WANT the preceding postprocessors to run; # so returning a intermediate result here instead of KeyboardInterrupt on live return result @@ -500,7 +490,6 @@ class FragmentFD(FileDownloader): download_fragment(fragment, ctx_copy) return fragment, fragment['frag_index'], ctx_copy.get('fragment_filename_sanitized') - self.report_warning('The download speed shown is only of one thread. This is a known issue') with tpe or concurrent.futures.ThreadPoolExecutor(max_workers) as pool: try: for fragment, frag_index, frag_filename in pool.map(_download_fragment, fragments): diff --git a/plugins/youtube_download/yt_dlp/downloader/hls.py b/plugins/youtube_download/yt_dlp/downloader/hls.py index d4b3f032..1f36a07f 100644 --- a/plugins/youtube_download/yt_dlp/downloader/hls.py +++ b/plugins/youtube_download/yt_dlp/downloader/hls.py @@ -16,6 +16,7 @@ from ..utils import ( update_url_query, urljoin, ) +from ..utils._utils import _request_dump_filename class HlsFD(FragmentFD): @@ -72,11 +73,23 @@ class HlsFD(FragmentFD): def real_download(self, filename, info_dict): man_url = info_dict['url'] - self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME) - urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url)) - man_url = urlh.url - s = urlh.read().decode('utf-8', 'ignore') + s = info_dict.get('hls_media_playlist_data') + if s: + self.to_screen(f'[{self.FD_NAME}] Using m3u8 manifest from extracted info') + else: + self.to_screen(f'[{self.FD_NAME}] Downloading m3u8 manifest') + urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url)) + man_url = urlh.url + s_bytes = urlh.read() + if self.params.get('write_pages'): + dump_filename = _request_dump_filename( + man_url, info_dict['id'], None, + trim_length=self.params.get('trim_file_name')) + self.to_screen(f'[{self.FD_NAME}] Saving request to {dump_filename}') + with open(dump_filename, 'wb') as outf: + outf.write(s_bytes) + s = s_bytes.decode('utf-8', 'ignore') can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None if can_download: @@ -119,12 +132,12 @@ class HlsFD(FragmentFD): self.to_screen(f'[{self.FD_NAME}] Fragment downloads will be delegated to {real_downloader.get_basename()}') def is_ad_fragment_start(s): - return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=ad' in s - or s.startswith('#UPLYNK-SEGMENT') and s.endswith(',ad')) + return ((s.startswith('#ANVATO-SEGMENT-INFO') and 'type=ad' in s) + or (s.startswith('#UPLYNK-SEGMENT') and s.endswith(',ad'))) def is_ad_fragment_end(s): - return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=master' in s - or s.startswith('#UPLYNK-SEGMENT') and s.endswith(',segment')) + return ((s.startswith('#ANVATO-SEGMENT-INFO') and 'type=master' in s) + or (s.startswith('#UPLYNK-SEGMENT') and s.endswith(',segment'))) fragments = [] @@ -160,10 +173,12 @@ class HlsFD(FragmentFD): extra_state = ctx.setdefault('extra_state', {}) format_index = info_dict.get('format_index') - extra_query = None - extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url') - if extra_param_to_segment_url: - extra_query = urllib.parse.parse_qs(extra_param_to_segment_url) + extra_segment_query = None + if extra_param_to_segment_url := info_dict.get('extra_param_to_segment_url'): + extra_segment_query = urllib.parse.parse_qs(extra_param_to_segment_url) + extra_key_query = None + if extra_param_to_key_url := info_dict.get('extra_param_to_key_url'): + extra_key_query = urllib.parse.parse_qs(extra_param_to_key_url) i = 0 media_sequence = 0 decrypt_info = {'METHOD': 'NONE'} @@ -175,6 +190,7 @@ class HlsFD(FragmentFD): if external_aes_iv: external_aes_iv = binascii.unhexlify(remove_start(external_aes_iv, '0x').zfill(32)) byte_range = {} + byte_range_offset = 0 discontinuity_count = 0 frag_index = 0 ad_frag_next = False @@ -190,8 +206,8 @@ class HlsFD(FragmentFD): if frag_index <= ctx['fragment_index']: continue frag_url = urljoin(man_url, line) - if extra_query: - frag_url = update_url_query(frag_url, extra_query) + if extra_segment_query: + frag_url = update_url_query(frag_url, extra_segment_query) fragments.append({ 'frag_index': frag_index, @@ -202,6 +218,11 @@ class HlsFD(FragmentFD): }) media_sequence += 1 + # If the byte_range is truthy, reset it after appending a fragment that uses it + if byte_range: + byte_range_offset = byte_range['end'] + byte_range = {} + elif line.startswith('#EXT-X-MAP'): if format_index and discontinuity_count != format_index: continue @@ -212,13 +233,15 @@ class HlsFD(FragmentFD): frag_index += 1 map_info = parse_m3u8_attributes(line[11:]) frag_url = urljoin(man_url, map_info.get('URI')) - if extra_query: - frag_url = update_url_query(frag_url, extra_query) + if extra_segment_query: + frag_url = update_url_query(frag_url, extra_segment_query) + + map_byte_range = {} if map_info.get('BYTERANGE'): splitted_byte_range = map_info.get('BYTERANGE').split('@') - sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end'] - byte_range = { + sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else 0 + map_byte_range = { 'start': sub_range_start, 'end': sub_range_start + int(splitted_byte_range[0]), } @@ -227,8 +250,8 @@ class HlsFD(FragmentFD): 'frag_index': frag_index, 'url': frag_url, 'decrypt_info': decrypt_info, - 'byte_range': byte_range, - 'media_sequence': media_sequence + 'byte_range': map_byte_range, + 'media_sequence': media_sequence, }) media_sequence += 1 @@ -244,8 +267,10 @@ class HlsFD(FragmentFD): decrypt_info['KEY'] = external_aes_key else: decrypt_info['URI'] = urljoin(man_url, decrypt_info['URI']) - if extra_query: - decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_query) + if extra_key_query or extra_segment_query: + # Fall back to extra_segment_query to key for backwards compat + decrypt_info['URI'] = update_url_query( + decrypt_info['URI'], extra_key_query or extra_segment_query) if decrypt_url != decrypt_info['URI']: decrypt_info['KEY'] = None @@ -253,7 +278,7 @@ class HlsFD(FragmentFD): media_sequence = int(line[22:]) elif line.startswith('#EXT-X-BYTERANGE'): splitted_byte_range = line[17:].split('@') - sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end'] + sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range_offset byte_range = { 'start': sub_range_start, 'end': sub_range_start + int(splitted_byte_range[0]), @@ -350,9 +375,8 @@ class HlsFD(FragmentFD): # XXX: this should probably be silent as well # or verify that all segments contain the same data self.report_warning(bug_reports_message( - 'Discarding a %s block found in the middle of the stream; ' - 'if the subtitles display incorrectly,' - % (type(block).__name__))) + f'Discarding a {type(block).__name__} block found in the middle of the stream; ' + 'if the subtitles display incorrectly,')) continue block.write_into(output) @@ -369,7 +393,10 @@ class HlsFD(FragmentFD): return output.getvalue().encode() - self.download_and_append_fragments( - ctx, fragments, info_dict, pack_func=pack_fragment, finish_func=fin_fragments) + if len(fragments) == 1: + self.download_and_append_fragments(ctx, fragments, info_dict) + else: + self.download_and_append_fragments( + ctx, fragments, info_dict, pack_func=pack_fragment, finish_func=fin_fragments) else: return self.download_and_append_fragments(ctx, fragments, info_dict) diff --git a/plugins/youtube_download/yt_dlp/downloader/http.py b/plugins/youtube_download/yt_dlp/downloader/http.py index f5237443..9c6dd8b7 100644 --- a/plugins/youtube_download/yt_dlp/downloader/http.py +++ b/plugins/youtube_download/yt_dlp/downloader/http.py @@ -15,7 +15,6 @@ from ..utils import ( ThrottledDownload, XAttrMetadataError, XAttrUnavailableError, - encodeFilename, int_or_none, parse_http_range, try_call, @@ -58,9 +57,8 @@ class HttpFD(FileDownloader): if self.params.get('continuedl', True): # Establish possible resume length - if os.path.isfile(encodeFilename(ctx.tmpfilename)): - ctx.resume_len = os.path.getsize( - encodeFilename(ctx.tmpfilename)) + if os.path.isfile(ctx.tmpfilename): + ctx.resume_len = os.path.getsize(ctx.tmpfilename) ctx.is_resume = ctx.resume_len > 0 @@ -176,7 +174,7 @@ class HttpFD(FileDownloader): 'downloaded_bytes': ctx.resume_len, 'total_bytes': ctx.resume_len, }, info_dict) - raise SucceedDownload() + raise SucceedDownload else: # The length does not match, we start the download over self.report_unable_to_resume() @@ -194,7 +192,7 @@ class HttpFD(FileDownloader): def close_stream(): if ctx.stream is not None: - if not ctx.tmpfilename == '-': + if ctx.tmpfilename != '-': ctx.stream.close() ctx.stream = None @@ -237,8 +235,13 @@ class HttpFD(FileDownloader): def retry(e): close_stream() - ctx.resume_len = (byte_counter if ctx.tmpfilename == '-' - else os.path.getsize(encodeFilename(ctx.tmpfilename))) + if ctx.tmpfilename == '-': + ctx.resume_len = byte_counter + else: + try: + ctx.resume_len = os.path.getsize(ctx.tmpfilename) + except FileNotFoundError: + ctx.resume_len = 0 raise RetryDownload(e) while True: @@ -263,20 +266,20 @@ class HttpFD(FileDownloader): ctx.filename = self.undo_temp_name(ctx.tmpfilename) self.report_destination(ctx.filename) except OSError as err: - self.report_error('unable to open for writing: %s' % str(err)) + self.report_error(f'unable to open for writing: {err}') return False if self.params.get('xattr_set_filesize', False) and data_len is not None: try: write_xattr(ctx.tmpfilename, 'user.ytdl.filesize', str(data_len).encode()) except (XAttrUnavailableError, XAttrMetadataError) as err: - self.report_error('unable to set filesize xattr: %s' % str(err)) + self.report_error(f'unable to set filesize xattr: {err}') try: ctx.stream.write(data_block) except OSError as err: self.to_stderr('\n') - self.report_error('unable to write data: %s' % str(err)) + self.report_error(f'unable to write data: {err}') return False # Apply rate limit @@ -322,7 +325,7 @@ class HttpFD(FileDownloader): elif now - ctx.throttle_start > 3: if ctx.stream is not None and ctx.tmpfilename != '-': ctx.stream.close() - raise ThrottledDownload() + raise ThrottledDownload elif speed: ctx.throttle_start = None @@ -333,7 +336,7 @@ class HttpFD(FileDownloader): if not is_test and ctx.chunk_size and ctx.content_len is not None and byte_counter < ctx.content_len: ctx.resume_len = byte_counter - raise NextFragment() + raise NextFragment if ctx.tmpfilename != '-': ctx.stream.close() diff --git a/plugins/youtube_download/yt_dlp/downloader/ism.py b/plugins/youtube_download/yt_dlp/downloader/ism.py index dd688f58..62c3a3b7 100644 --- a/plugins/youtube_download/yt_dlp/downloader/ism.py +++ b/plugins/youtube_download/yt_dlp/downloader/ism.py @@ -251,7 +251,7 @@ class IsmFD(FragmentFD): skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True) frag_index = 0 - for i, segment in enumerate(segments): + for segment in segments: frag_index += 1 if frag_index <= ctx['fragment_index']: continue diff --git a/plugins/youtube_download/yt_dlp/downloader/mhtml.py b/plugins/youtube_download/yt_dlp/downloader/mhtml.py index d977dcec..3d4f2d76 100644 --- a/plugins/youtube_download/yt_dlp/downloader/mhtml.py +++ b/plugins/youtube_download/yt_dlp/downloader/mhtml.py @@ -10,7 +10,7 @@ from ..version import __version__ as YT_DLP_VERSION class MhtmlFD(FragmentFD): - _STYLESHEET = """\ + _STYLESHEET = '''\ html, body { margin: 0; padding: 0; @@ -45,7 +45,7 @@ body > figure > img { max-width: 100%; max-height: calc(100vh - 5em); } -""" +''' _STYLESHEET = re.sub(r'\s+', ' ', _STYLESHEET) _STYLESHEET = re.sub(r'\B \B|(?<=[\w\-]) (?=[^\w\-])|(?<=[^\w\-]) (?=[\w\-])', '', _STYLESHEET) @@ -57,24 +57,19 @@ body > figure > img { )).decode('us-ascii') + '?=' def _gen_cid(self, i, fragment, frag_boundary): - return '%u.%s@yt-dlp.github.io.invalid' % (i, frag_boundary) + return f'{i}.{frag_boundary}@yt-dlp.github.io.invalid' def _gen_stub(self, *, fragments, frag_boundary, title): output = io.StringIO() - output.write(( + output.write( '' '' '' - '' '' - '' '{title}' - '' '' - '' - ).format( - version=escapeHTML(YT_DLP_VERSION), - styles=self._STYLESHEET, - title=escapeHTML(title) - )) + f'' + f'{escapeHTML(title)}' + f'' + '') t0 = 0 for i, frag in enumerate(fragments): @@ -87,15 +82,12 @@ body > figure > img { num=i + 1, t0=srt_subtitles_timecode(t0), t1=srt_subtitles_timecode(t1), - duration=formatSeconds(frag['duration'], msec=True) + duration=formatSeconds(frag['duration'], msec=True), )) except (KeyError, ValueError, TypeError): t1 = None - output.write(( - '
Slide #{num}
' - ).format(num=i + 1)) - output.write(''.format( - cid=self._gen_cid(i, frag, frag_boundary))) + output.write(f'
Slide #{i + 1}
') + output.write(f'') output.write('') t0 = t1 @@ -126,31 +118,24 @@ body > figure > img { stub = self._gen_stub( fragments=fragments, frag_boundary=frag_boundary, - title=title + title=title, ) ctx['dest_stream'].write(( 'MIME-Version: 1.0\r\n' 'From: \r\n' 'To: \r\n' - 'Subject: {title}\r\n' + f'Subject: {self._escape_mime(title)}\r\n' 'Content-type: multipart/related; ' - '' 'boundary="{boundary}"; ' - '' 'type="text/html"\r\n' - 'X.yt-dlp.Origin: {origin}\r\n' + f'boundary="{frag_boundary}"; ' + 'type="text/html"\r\n' + f'X.yt-dlp.Origin: {origin}\r\n' '\r\n' - '--{boundary}\r\n' + f'--{frag_boundary}\r\n' 'Content-Type: text/html; charset=utf-8\r\n' - 'Content-Length: {length}\r\n' + f'Content-Length: {len(stub)}\r\n' '\r\n' - '{stub}\r\n' - ).format( - origin=origin, - boundary=frag_boundary, - length=len(stub), - title=self._escape_mime(title), - stub=stub - ).encode()) + f'{stub}\r\n').encode()) extra_state['header_written'] = True for i, fragment in enumerate(fragments): diff --git a/plugins/youtube_download/yt_dlp/downloader/niconico.py b/plugins/youtube_download/yt_dlp/downloader/niconico.py index 5720f6eb..33cf15df 100644 --- a/plugins/youtube_download/yt_dlp/downloader/niconico.py +++ b/plugins/youtube_download/yt_dlp/downloader/niconico.py @@ -2,58 +2,10 @@ import json import threading import time -from . import get_suitable_downloader from .common import FileDownloader from .external import FFmpegFD from ..networking import Request -from ..utils import DownloadError, WebSocketsWrapper, str_or_none, try_get - - -class NiconicoDmcFD(FileDownloader): - """ Downloading niconico douga from DMC with heartbeat """ - - def real_download(self, filename, info_dict): - from ..extractor.niconico import NiconicoIE - - self.to_screen('[%s] Downloading from DMC' % self.FD_NAME) - ie = NiconicoIE(self.ydl) - info_dict, heartbeat_info_dict = ie._get_heartbeat_info(info_dict) - - fd = get_suitable_downloader(info_dict, params=self.params)(self.ydl, self.params) - - success = download_complete = False - timer = [None] - heartbeat_lock = threading.Lock() - heartbeat_url = heartbeat_info_dict['url'] - heartbeat_data = heartbeat_info_dict['data'].encode() - heartbeat_interval = heartbeat_info_dict.get('interval', 30) - - request = Request(heartbeat_url, heartbeat_data) - - def heartbeat(): - try: - self.ydl.urlopen(request).read() - except Exception: - self.to_screen('[%s] Heartbeat failed' % self.FD_NAME) - - with heartbeat_lock: - if not download_complete: - timer[0] = threading.Timer(heartbeat_interval, heartbeat) - timer[0].start() - - heartbeat_info_dict['ping']() - self.to_screen('[%s] Heartbeat with %d second interval ...' % (self.FD_NAME, heartbeat_interval)) - try: - heartbeat() - if type(fd).__name__ == 'HlsFD': - info_dict.update(ie._extract_m3u8_formats(info_dict['url'], info_dict['id'])[0]) - success = fd.real_download(filename, info_dict) - finally: - if heartbeat_lock: - with heartbeat_lock: - timer[0].cancel() - download_complete = True - return success +from ..utils import DownloadError, str_or_none, try_get class NiconicoLiveFD(FileDownloader): @@ -64,7 +16,6 @@ class NiconicoLiveFD(FileDownloader): ws_url = info_dict['url'] ws_extractor = info_dict['ws'] ws_origin_host = info_dict['origin'] - cookies = info_dict.get('cookies') live_quality = info_dict.get('live_quality', 'high') live_latency = info_dict.get('live_latency', 'high') dl = FFmpegFD(self.ydl, self.params or {}) @@ -76,12 +27,7 @@ class NiconicoLiveFD(FileDownloader): def communicate_ws(reconnect): if reconnect: - ws = WebSocketsWrapper(ws_url, { - 'Cookies': str_or_none(cookies) or '', - 'Origin': f'https://{ws_origin_host}', - 'Accept': '*/*', - 'User-Agent': self.params['http_headers']['User-Agent'], - }) + ws = self.ydl.urlopen(Request(ws_url, headers={'Origin': f'https://{ws_origin_host}'})) if self.ydl.params.get('verbose', False): self.to_screen('[debug] Sending startWatching request') ws.send(json.dumps({ @@ -91,14 +37,15 @@ class NiconicoLiveFD(FileDownloader): 'quality': live_quality, 'protocol': 'hls+fmp4', 'latency': live_latency, - 'chasePlay': False + 'accessRightMethod': 'single_cookie', + 'chasePlay': False, }, 'room': { 'protocol': 'webSocket', - 'commentable': True + 'commentable': True, }, 'reconnect': True, - } + }, })) else: ws = ws_extractor @@ -124,7 +71,7 @@ class NiconicoLiveFD(FileDownloader): elif self.ydl.params.get('verbose', False): if len(recv) > 100: recv = recv[:100] + '...' - self.to_screen('[debug] Server said: %s' % recv) + self.to_screen(f'[debug] Server said: {recv}') def ws_main(): reconnect = False @@ -134,7 +81,7 @@ class NiconicoLiveFD(FileDownloader): if ret is True: return except BaseException as e: - self.to_screen('[%s] %s: Connection error occured, reconnecting after 10 seconds: %s' % ('niconico:live', video_id, str_or_none(e))) + self.to_screen('[{}] {}: Connection error occured, reconnecting after 10 seconds: {}'.format('niconico:live', video_id, str_or_none(e))) time.sleep(10) continue finally: diff --git a/plugins/youtube_download/yt_dlp/downloader/rtmp.py b/plugins/youtube_download/yt_dlp/downloader/rtmp.py index 0e095259..1b831e5f 100644 --- a/plugins/youtube_download/yt_dlp/downloader/rtmp.py +++ b/plugins/youtube_download/yt_dlp/downloader/rtmp.py @@ -8,7 +8,6 @@ from ..utils import ( Popen, check_executable, encodeArgument, - encodeFilename, get_exe_version, ) @@ -179,15 +178,15 @@ class RtmpFD(FileDownloader): return False while retval in (RD_INCOMPLETE, RD_FAILED) and not test and not live: - prevsize = os.path.getsize(encodeFilename(tmpfilename)) - self.to_screen('[rtmpdump] Downloaded %s bytes' % prevsize) + prevsize = os.path.getsize(tmpfilename) + self.to_screen(f'[rtmpdump] Downloaded {prevsize} bytes') time.sleep(5.0) # This seems to be needed - args = basic_args + ['--resume'] + args = [*basic_args, '--resume'] if retval == RD_FAILED: args += ['--skip', '1'] args = [encodeArgument(a) for a in args] retval = run_rtmpdump(args) - cursize = os.path.getsize(encodeFilename(tmpfilename)) + cursize = os.path.getsize(tmpfilename) if prevsize == cursize and retval == RD_FAILED: break # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those @@ -196,8 +195,8 @@ class RtmpFD(FileDownloader): retval = RD_SUCCESS break if retval == RD_SUCCESS or (test and retval == RD_INCOMPLETE): - fsize = os.path.getsize(encodeFilename(tmpfilename)) - self.to_screen('[rtmpdump] Downloaded %s bytes' % fsize) + fsize = os.path.getsize(tmpfilename) + self.to_screen(f'[rtmpdump] Downloaded {fsize} bytes') self.try_rename(tmpfilename, filename) self._hook_progress({ 'downloaded_bytes': fsize, diff --git a/plugins/youtube_download/yt_dlp/downloader/rtsp.py b/plugins/youtube_download/yt_dlp/downloader/rtsp.py index e89269fe..b4b0be7e 100644 --- a/plugins/youtube_download/yt_dlp/downloader/rtsp.py +++ b/plugins/youtube_download/yt_dlp/downloader/rtsp.py @@ -2,7 +2,7 @@ import os import subprocess from .common import FileDownloader -from ..utils import check_executable, encodeFilename +from ..utils import check_executable class RtspFD(FileDownloader): @@ -26,7 +26,7 @@ class RtspFD(FileDownloader): retval = subprocess.call(args) if retval == 0: - fsize = os.path.getsize(encodeFilename(tmpfilename)) + fsize = os.path.getsize(tmpfilename) self.to_screen(f'\r[{args[0]}] {fsize} bytes') self.try_rename(tmpfilename, filename) self._hook_progress({ diff --git a/plugins/youtube_download/yt_dlp/downloader/youtube_live_chat.py b/plugins/youtube_download/yt_dlp/downloader/youtube_live_chat.py index c7a86374..ddd912ca 100644 --- a/plugins/youtube_download/yt_dlp/downloader/youtube_live_chat.py +++ b/plugins/youtube_download/yt_dlp/downloader/youtube_live_chat.py @@ -18,7 +18,7 @@ class YoutubeLiveChatFD(FragmentFD): def real_download(self, filename, info_dict): video_id = info_dict['video_id'] - self.to_screen('[%s] Downloading live chat' % self.FD_NAME) + self.to_screen(f'[{self.FD_NAME}] Downloading live chat') if not self.params.get('skip_download') and info_dict['protocol'] == 'youtube_live_chat': self.report_warning('Live chat download runs until the livestream ends. ' 'If you wish to download the video simultaneously, run a separate yt-dlp instance') @@ -123,8 +123,8 @@ class YoutubeLiveChatFD(FragmentFD): data, lambda x: x['continuationContents']['liveChatContinuation'], dict) or {} - func = (info_dict['protocol'] == 'youtube_live_chat' and parse_actions_live - or frag_index == 1 and try_refresh_replay_beginning + func = ((info_dict['protocol'] == 'youtube_live_chat' and parse_actions_live) + or (frag_index == 1 and try_refresh_replay_beginning) or parse_actions_replay) return (True, *func(live_chat_continuation)) except HTTPError as err: diff --git a/plugins/youtube_download/yt_dlp/extractor/__init__.py b/plugins/youtube_download/yt_dlp/extractor/__init__.py index 6bfa4bd7..a090e942 100644 --- a/plugins/youtube_download/yt_dlp/extractor/__init__.py +++ b/plugins/youtube_download/yt_dlp/extractor/__init__.py @@ -1,16 +1,25 @@ from ..compat.compat_utils import passthrough_module +from ..globals import extractors as _extractors_context +from ..globals import plugin_ies as _plugin_ies_context +from ..plugins import PluginSpec, register_plugin_spec passthrough_module(__name__, '.extractors') del passthrough_module +register_plugin_spec(PluginSpec( + module_name='extractor', + suffix='IE', + destination=_extractors_context, + plugin_destination=_plugin_ies_context, +)) + def gen_extractor_classes(): """ Return a list of supported extractors. The order does matter; the first extractor matched is the one handling the URL. """ - from .extractors import _ALL_CLASSES - - return _ALL_CLASSES + import_extractors() + return list(_extractors_context.value.values()) def gen_extractors(): @@ -37,6 +46,9 @@ def list_extractors(age_limit=None): def get_info_extractor(ie_name): """Returns the info extractor class with the given ie_name""" - from . import extractors + import_extractors() + return _extractors_context.value[f'{ie_name}IE'] - return getattr(extractors, f'{ie_name}IE') + +def import_extractors(): + from . import extractors # noqa: F401 diff --git a/plugins/youtube_download/yt_dlp/extractor/_extractors.py b/plugins/youtube_download/yt_dlp/extractor/_extractors.py index 63bb55ea..bb1c3db1 100644 --- a/plugins/youtube_download/yt_dlp/extractor/_extractors.py +++ b/plugins/youtube_download/yt_dlp/extractor/_extractors.py @@ -1,4 +1,5 @@ # flake8: noqa: F401 +# isort: off from .youtube import ( # Youtube is moved to the top to improve performance YoutubeIE, @@ -24,6 +25,8 @@ from .youtube import ( # Youtube is moved to the top to improve performance YoutubeConsentRedirectIE, ) +# isort: on + from .abc import ( ABCIE, ABCIViewIE, @@ -43,30 +46,37 @@ from .abematv import ( ) from .academicearth import AcademicEarthCourseIE from .acast import ( - ACastIE, ACastChannelIE, + ACastIE, +) +from .acfun import ( + AcFunBangumiIE, + AcFunVideoIE, +) +from .adn import ( + ADNIE, + ADNSeasonIE, ) -from .acfun import AcFunVideoIE, AcFunBangumiIE -from .adn import ADNIE from .adobeconnect import AdobeConnectIE from .adobetv import ( + AdobeTVChannelIE, AdobeTVEmbedIE, AdobeTVIE, AdobeTVShowIE, - AdobeTVChannelIE, AdobeTVVideoIE, ) from .adultswim import AdultSwimIE from .aenetworks import ( - AENetworksIE, AENetworksCollectionIE, + AENetworksIE, AENetworksShowIE, - HistoryTopicIE, - HistoryPlayerIE, BiographyIE, + HistoryPlayerIE, + HistoryTopicIE, ) from .aeonco import AeonCoIE from .afreecatv import ( + AfreecaTVCatchStoryIE, AfreecaTVIE, AfreecaTVLiveIE, AfreecaTVUserIE, @@ -77,74 +87,90 @@ from .agora import ( WyborczaPodcastIE, WyborczaVideoIE, ) -from .airmozilla import AirMozillaIE from .airtv import AirTVIE from .aitube import AitubeKZVideoIE +from .aliexpress import AliExpressLiveIE from .aljazeera import AlJazeeraIE -from .alphaporno import AlphaPornoIE -from .amara import AmaraIE -from .alura import ( - AluraIE, - AluraCourseIE +from .allocine import AllocineIE +from .allstar import ( + AllstarIE, + AllstarProfileIE, ) -from .amcnetworks import AMCNetworksIE +from .alphaporno import AlphaPornoIE +from .alsace20tv import ( + Alsace20TVEmbedIE, + Alsace20TVIE, +) +from .altcensored import ( + AltCensoredChannelIE, + AltCensoredIE, +) +from .alura import ( + AluraCourseIE, + AluraIE, +) +from .amadeustv import AmadeusTVIE +from .amara import AmaraIE from .amazon import ( - AmazonStoreIE, AmazonReviewsIE, + AmazonStoreIE, ) from .amazonminitv import ( AmazonMiniTVIE, AmazonMiniTVSeasonIE, AmazonMiniTVSeriesIE, ) +from .amcnetworks import AMCNetworksIE from .americastestkitchen import ( AmericasTestKitchenIE, AmericasTestKitchenSeasonIE, ) from .anchorfm import AnchorFMEpisodeIE from .angel import AngelIE +from .antenna import ( + Ant1NewsGrArticleIE, + Ant1NewsGrEmbedIE, + AntennaGrWatchIE, +) from .anvato import AnvatoIE from .aol import AolIE -from .allocine import AllocineIE -from .aliexpress import AliExpressLiveIE -from .alsace20tv import ( - Alsace20TVIE, - Alsace20TVEmbedIE, -) from .apa import APAIE from .aparat import AparatIE from .appleconnect import AppleConnectIE +from .applepodcasts import ApplePodcastsIE from .appletrailers import ( AppleTrailersIE, AppleTrailersSectionIE, ) -from .applepodcasts import ApplePodcastsIE from .archiveorg import ( ArchiveOrgIE, YoutubeWebArchiveIE, - VLiveWebArchiveIE, ) from .arcpublishing import ArcPublishingIE -from .arkena import ArkenaIE from .ard import ( - ARDBetaMediathekIE, ARDIE, - ARDMediathekIE, + ARDBetaMediathekIE, + ARDMediathekCollectionIE, +) +from .arkena import ArkenaIE +from .arnes import ArnesIE +from .art19 import ( + Art19IE, + Art19ShowIE, ) from .arte import ( - ArteTVIE, - ArteTVEmbedIE, - ArteTVPlaylistIE, ArteTVCategoryIE, + ArteTVEmbedIE, + ArteTVIE, + ArteTVPlaylistIE, ) -from .arnes import ArnesIE -from .asiancrush import ( - AsianCrushIE, - AsianCrushPlaylistIE, +from .asobichannel import ( + AsobiChannelIE, + AsobiChannelTagURLIE, ) +from .asobistage import AsobiStageIE from .atresplayer import AtresPlayerIE from .atscaleconf import AtScaleConfEventIE -from .atttechchannel import ATTTechChannelIE from .atvat import ATVAtIE from .audimedia import AudiMediaIE from .audioboom import AudioBoomIE @@ -152,56 +178,65 @@ from .audiodraft import ( AudiodraftCustomIE, AudiodraftGenericIE, ) -from .audiomack import AudiomackIE, AudiomackAlbumIE +from .audiomack import ( + AudiomackAlbumIE, + AudiomackIE, +) from .audius import ( AudiusIE, - AudiusTrackIE, AudiusPlaylistIE, AudiusProfileIE, + AudiusTrackIE, ) from .awaan import ( AWAANIE, - AWAANVideoIE, AWAANLiveIE, AWAANSeasonIE, + AWAANVideoIE, ) +from .axs import AxsIE from .azmedien import AZMedienIE from .baidu import BaiduVideoIE from .banbye import ( - BanByeIE, BanByeChannelIE, + BanByeIE, ) from .bandaichannel import BandaiChannelIE from .bandcamp import ( - BandcampIE, BandcampAlbumIE, - BandcampWeeklyIE, + BandcampIE, BandcampUserIE, + BandcampWeeklyIE, +) +from .bandlab import ( + BandlabIE, + BandlabPlaylistIE, ) from .bannedvideo import BannedVideoIE from .bbc import ( - BBCCoUkIE, + BBCIE, BBCCoUkArticleIE, + BBCCoUkIE, BBCCoUkIPlayerEpisodesIE, BBCCoUkIPlayerGroupIE, BBCCoUkPlaylistIE, - BBCIE, ) +from .beacon import BeaconTvIE +from .beatbump import ( + BeatBumpPlaylistIE, + BeatBumpVideoIE, +) +from .beatport import BeatportIE from .beeg import BeegIE from .behindkink import BehindKinkIE from .bellmedia import BellMediaIE -from .beatbump import ( - BeatBumpVideoIE, - BeatBumpPlaylistIE, -) -from .beatport import BeatportIE from .berufetv import BerufeTVIE from .bet import BetIE from .bfi import BFIPlayerIE from .bfmtv import ( BFMTVIE, - BFMTVLiveIE, BFMTVArticleIE, + BFMTVLiveIE, ) from .bibeltv import ( BibelTVLiveIE, @@ -212,112 +247,116 @@ from .bigflix import BigflixIE from .bigo import BigoIE from .bild import BildIE from .bilibili import ( - BiliBiliIE, - BiliBiliBangumiIE, - BiliBiliBangumiSeasonIE, - BiliBiliBangumiMediaIE, - BiliBiliSearchIE, - BilibiliCategoryIE, - BilibiliAudioIE, BilibiliAudioAlbumIE, + BilibiliAudioIE, + BiliBiliBangumiIE, + BiliBiliBangumiMediaIE, + BiliBiliBangumiSeasonIE, + BilibiliCategoryIE, + BilibiliCheeseIE, + BilibiliCheeseSeasonIE, + BilibiliCollectionListIE, + BiliBiliDynamicIE, + BilibiliFavoritesListIE, + BiliBiliIE, BiliBiliPlayerIE, - BilibiliSpaceVideoIE, + BilibiliPlaylistIE, + BiliBiliSearchIE, + BilibiliSeriesListIE, BilibiliSpaceAudioIE, - BilibiliSpacePlaylistIE, + BilibiliSpaceVideoIE, + BilibiliWatchlaterIE, BiliIntlIE, BiliIntlSeriesIE, BiliLiveIE, ) from .biobiochiletv import BioBioChileTVIE from .bitchute import ( - BitChuteIE, BitChuteChannelIE, + BitChuteIE, ) -from .bitwave import ( - BitwaveReplayIE, - BitwaveStreamIE, -) -from .biqle import BIQLEIE from .blackboardcollaborate import BlackboardCollaborateIE from .bleacherreport import ( - BleacherReportIE, BleacherReportCMSIE, + BleacherReportIE, ) from .blerp import BlerpIE from .blogger import BloggerIE from .bloomberg import BloombergIE +from .bluesky import BlueskyIE from .bokecc import BokeCCIE from .bongacams import BongaCamsIE +from .boosty import BoostyIE from .bostonglobe import BostonGlobeIE from .box import BoxIE from .boxcast import BoxCastVideoIE from .bpb import BpbIE -from .br import ( - BRIE, - BRMediathekIE, -) -from .bravotv import BravoTVIE +from .br import BRIE from .brainpop import ( - BrainPOPIE, - BrainPOPJrIE, BrainPOPELLIE, BrainPOPEspIE, BrainPOPFrIE, + BrainPOPIE, BrainPOPIlIE, + BrainPOPJrIE, ) -from .breakcom import BreakIE +from .bravotv import BravoTVIE from .breitbart import BreitBartIE from .brightcove import ( BrightcoveLegacyIE, BrightcoveNewIE, ) -from .businessinsider import BusinessInsiderIE +from .brilliantpala import ( + BrilliantpalaClassesIE, + BrilliantpalaElearnIE, +) from .bundesliga import BundesligaIE +from .bundestag import BundestagIE +from .bunnycdn import BunnyCdnIE +from .businessinsider import BusinessInsiderIE from .buzzfeed import BuzzFeedIE from .byutv import BYUtvIE from .c56 import C56IE -from .cableav import CableAVIE +from .caffeinetv import CaffeineTVIE from .callin import CallinIE from .caltrans import CaltransIE from .cam4 import CAM4IE from .camdemy import ( + CamdemyFolderIE, CamdemyIE, - CamdemyFolderIE ) from .camfm import ( CamFMEpisodeIE, - CamFMShowIE + CamFMShowIE, ) from .cammodels import CamModelsIE from .camsoda import CamsodaIE from .camtasia import CamtasiaEmbedIE -from .camwithher import CamWithHerIE +from .canal1 import Canal1IE from .canalalpha import CanalAlphaIE -from .canalplus import CanalplusIE from .canalc2 import Canalc2IE -from .carambatv import ( - CarambaTVIE, - CarambaTVPageIE, -) +from .canalplus import CanalplusIE +from .canalsurmas import CanalsurmasIE +from .caracoltv import CaracolTvPlayIE from .cartoonnetwork import CartoonNetworkIE from .cbc import ( CBCIE, - CBCPlayerIE, CBCGemIE, - CBCGemPlaylistIE, CBCGemLiveIE, + CBCGemPlaylistIE, + CBCPlayerIE, + CBCPlayerPlaylistIE, ) from .cbs import ( CBSIE, ParamountPressExpressIE, ) -from .cbsinteractive import CBSInteractiveIE from .cbsnews import ( + CBSLocalArticleIE, + CBSLocalIE, + CBSLocalLiveIE, CBSNewsEmbedIE, CBSNewsIE, - CBSLocalIE, - CBSLocalArticleIE, - CBSLocalLiveIE, CBSNewsLiveIE, CBSNewsLiveVideoIE, ) @@ -332,58 +371,55 @@ from .ccc import ( ) from .ccma import CCMAIE from .cctv import CCTVIE -from .cda import CDAIE +from .cda import ( + CDAIE, + CDAFolderIE, +) from .cellebrite import CellebriteIE from .ceskatelevize import CeskaTelevizeIE from .cgtn import CGTNIE -from .channel9 import Channel9IE from .charlierose import CharlieRoseIE from .chaturbate import ChaturbateIE from .chilloutzone import ChilloutzoneIE -from .chingari import ( - ChingariIE, - ChingariUserIE, +from .chzzk import ( + CHZZKLiveIE, + CHZZKVideoIE, ) -from .chirbit import ( - ChirbitIE, - ChirbitProfileIE, -) -from .cinchcast import CinchcastIE from .cinemax import CinemaxIE from .cinetecamilano import CinetecaMilanoIE +from .cineverse import ( + CineverseDetailsIE, + CineverseIE, +) from .ciscolive import ( - CiscoLiveSessionIE, CiscoLiveSearchIE, + CiscoLiveSessionIE, ) from .ciscowebex import CiscoWebexIE from .cjsw import CJSWIE from .clipchamp import ClipchampIE -from .cliphunter import CliphunterIE from .clippit import ClippitIE from .cliprs import ClipRsIE -from .clipsyndicate import ClipsyndicateIE from .closertotruth import CloserToTruthIE from .cloudflarestream import CloudflareStreamIE -from .cloudy import CloudyIE +from .cloudycdn import CloudyCDNIE from .clubic import ClubicIE from .clyp import ClypIE from .cmt import CMTIE -from .cnbc import ( - CNBCIE, - CNBCVideoIE, -) +from .cnbc import CNBCVideoIE from .cnn import ( CNNIE, - CNNBlogsIE, - CNNArticleIE, CNNIndonesiaIE, ) -from .coub import CoubIE from .comedycentral import ( ComedyCentralIE, ComedyCentralTVIE, ) -from .commonmistakes import CommonMistakesIE, UnicodeBOMIE +from .commonmistakes import ( + BlobIE, + CommonMistakesIE, + UnicodeBOMIE, +) from .commonprotocols import ( MmsIE, RtmpIE, @@ -392,50 +428,51 @@ from .commonprotocols import ( from .condenast import CondeNastIE from .contv import CONtvIE from .corus import CorusIE +from .coub import CoubIE +from .cozytv import CozyTVIE from .cpac import ( CPACIE, CPACPlaylistIE, ) -from .cozytv import CozyTVIE from .cracked import CrackedIE from .crackle import CrackleIE from .craftsy import CraftsyIE from .crooksandliars import CrooksAndLiarsIE from .crowdbunker import ( - CrowdBunkerIE, CrowdBunkerChannelIE, + CrowdBunkerIE, ) from .crtvg import CrtvgIE -from .crunchyroll import ( - CrunchyrollBetaIE, - CrunchyrollBetaShowIE, - CrunchyrollMusicIE, - CrunchyrollArtistIE, +from .cspan import ( + CSpanCongressIE, + CSpanIE, ) -from .cspan import CSpanIE, CSpanCongressIE from .ctsnews import CtsNewsIE from .ctv import CTVIE from .ctvnews import CTVNewsIE from .cultureunplugged import CultureUnpluggedIE from .curiositystream import ( - CuriosityStreamIE, CuriosityStreamCollectionsIE, + CuriosityStreamIE, CuriosityStreamSeriesIE, ) -from .cwtv import CWTVIE +from .cwtv import ( + CWTVIE, + CWTVMovieIE, +) from .cybrary import ( + CybraryCourseIE, CybraryIE, - CybraryCourseIE ) from .dacast import ( - DacastVODIE, DacastPlaylistIE, + DacastVODIE, ) -from .daftsex import DaftsexIE from .dailymail import DailyMailIE from .dailymotion import ( DailymotionIE, DailymotionPlaylistIE, + DailymotionSearchIE, DailymotionUserIE, ) from .dailywire import ( @@ -446,61 +483,76 @@ from .damtomo import ( DamtomoRecordIE, DamtomoVideoIE, ) +from .dangalplay import ( + DangalPlayIE, + DangalPlaySeasonIE, +) from .daum import ( - DaumIE, DaumClipIE, + DaumIE, DaumPlaylistIE, DaumUserIE, ) from .daystar import DaystarClipIE from .dbtv import DBTVIE from .dctp import DctpTvIE -from .deezer import ( - DeezerPlaylistIE, - DeezerAlbumIE, -) from .democracynow import DemocracynowIE from .detik import DetikEmbedIE +from .deuxm import ( + DeuxMIE, + DeuxMNewsIE, +) +from .dfb import DFBIE +from .dhm import DHMIE +from .digitalconcerthall import DigitalConcertHallIE +from .digiteka import DigitekaIE +from .digiview import DigiviewIE +from .discogs import DiscogsReleasePlaylistIE +from .disney import DisneyIE +from .dispeak import DigitallySpeakingIE from .dlf import ( DLFIE, DLFCorpusIE, ) -from .dfb import DFBIE -from .dhm import DHMIE -from .digg import DiggIE -from .dotsub import DotsubIE +from .dlive import ( + DLiveStreamIE, + DLiveVODIE, +) from .douyutv import ( DouyuShowIE, DouyuTVIE, ) from .dplay import ( - DPlayIE, - DiscoveryPlusIE, - HGTVDeIE, - GoDiscoveryIE, - TravelChannelIE, - CookingChannelIE, - HGTVUsaIE, - FoodNetworkIE, - InvestigationDiscoveryIE, - DestinationAmericaIE, - AmHistoryChannelIE, - ScienceChannelIE, - DIYNetworkIE, - DiscoveryLifeIE, - AnimalPlanetIE, TLCIE, - MotorTrendIE, - MotorTrendOnDemandIE, - DiscoveryPlusIndiaIE, + AmHistoryChannelIE, + AnimalPlanetIE, + CookingChannelIE, + DestinationAmericaIE, + DiscoveryLifeIE, DiscoveryNetworksDeIE, + DiscoveryPlusIE, + DiscoveryPlusIndiaIE, + DiscoveryPlusIndiaShowIE, DiscoveryPlusItalyIE, DiscoveryPlusItalyShowIE, - DiscoveryPlusIndiaShowIE, - GlobalCyclingNetworkPlusIE, + DPlayIE, + FoodNetworkIE, + GoDiscoveryIE, + HGTVDeIE, + HGTVUsaIE, + InvestigationDiscoveryIE, + ScienceChannelIE, + TravelChannelIE, ) -from .dreisat import DreiSatIE from .drbonanza import DRBonanzaIE +from .dreisat import DreiSatIE +from .drooble import DroobleIE +from .dropbox import DropboxIE +from .dropout import ( + DropoutIE, + DropoutSeasonIE, +) +from .drtalks import DrTalksIE from .drtuber import DrTuberIE from .drtv import ( DRTVIE, @@ -509,97 +561,86 @@ from .drtv import ( DRTVSeriesIE, ) from .dtube import DTubeIE -from .dvtv import DVTVIE from .duboku import ( DubokuIE, - DubokuPlaylistIE + DubokuPlaylistIE, ) from .dumpert import DumpertIE -from .defense import DefenseGouvFrIE -from .deuxm import ( - DeuxMIE, - DeuxMNewsIE -) -from .digitalconcerthall import DigitalConcertHallIE -from .discogs import DiscogsReleasePlaylistIE -from .discovery import DiscoveryIE -from .disney import DisneyIE -from .dispeak import DigitallySpeakingIE -from .dropbox import DropboxIE -from .dropout import ( - DropoutSeasonIE, - DropoutIE -) +from .duoplay import DuoplayIE +from .dvtv import DVTVIE from .dw import ( DWIE, DWArticleIE, ) -from .eagleplatform import EaglePlatformIE, ClipYouEmbedIE +from .eagleplatform import ( + ClipYouEmbedIE, + EaglePlatformIE, +) from .ebaumsworld import EbaumsWorldIE from .ebay import EbayIE -from .echomsk import EchoMskIE from .egghead import ( EggheadCourseIE, EggheadLessonIE, ) -from .ehow import EHowIE -from .eighttracks import EightTracksIE -from .einthusan import EinthusanIE -from .eitb import EitbIE -from .elevensports import ElevenSportsIE -from .ellentube import ( - EllenTubeIE, - EllenTubeVideoIE, - EllenTubePlaylistIE, +from .eggs import ( + EggsArtistIE, + EggsIE, ) +from .eighttracks import EightTracksIE +from .eitb import EitbIE +from .elementorembed import ElementorEmbedIE from .elonet import ElonetIE from .elpais import ElPaisIE +from .eltrecetv import ElTreceTVIE from .embedly import EmbedlyIE -from .engadget import EngadgetIE from .epicon import ( EpiconIE, EpiconSeriesIE, ) +from .epidemicsound import EpidemicSoundIE +from .eplus import EplusIbIE from .epoch import EpochIE from .eporner import EpornerIE +from .erocast import ErocastIE from .eroprofile import ( - EroProfileIE, EroProfileAlbumIE, + EroProfileIE, ) +from .err import ERRJupiterIE from .ertgr import ( ERTFlixCodenameIE, ERTFlixIE, ERTWebtvEmbedIE, ) -from .escapist import EscapistIE from .espn import ( ESPNIE, - WatchESPNIE, ESPNArticleIE, - FiveThirtyEightIE, ESPNCricInfoIE, + FiveThirtyEightIE, + WatchESPNIE, ) -from .esri import EsriVideoIE from .ettutv import EttuTvIE -from .europa import EuropaIE, EuroParlWebstreamIE +from .europa import ( + EuropaIE, + EuroParlWebstreamIE, +) from .europeantour import EuropeanTourIE from .eurosport import EurosportIE from .euscreen import EUScreenIE -from .expotv import ExpoTVIE from .expressen import ExpressenIE -from .extremetube import ExtremeTubeIE from .eyedotv import EyedoTVIE from .facebook import ( + FacebookAdsIE, FacebookIE, FacebookPluginsVideoIE, FacebookRedirectURLIE, FacebookReelIE, ) from .fancode import ( + FancodeLiveIE, FancodeVodIE, - FancodeLiveIE ) - +from .fathom import FathomIE from .faz import FazIE from .fc2 import ( FC2IE, @@ -608,136 +649,144 @@ from .fc2 import ( ) from .fczenit import FczenitIE from .fifa import FifaIE -from .filmmodu import FilmmoduIE from .filmon import ( - FilmOnIE, FilmOnChannelIE, + FilmOnIE, ) from .filmweb import FilmwebIE from .firsttv import FirstTVIE from .fivetv import FiveTVIE +from .flextv import FlexTVIE from .flickr import FlickrIE +from .floatplane import ( + FloatplaneChannelIE, + FloatplaneIE, +) from .folketinget import FolketingetIE from .footyroom import FootyRoomIE from .formula1 import Formula1IE from .fourtube import ( FourTubeIE, - PornTubeIE, - PornerBrosIE, FuxIE, -) -from .fourzerostudio import ( - FourZeroStudioArchiveIE, - FourZeroStudioClipIE, + PornerBrosIE, + PornTubeIE, ) from .fox import FOXIE from .fox9 import ( FOX9IE, FOX9NewsIE, ) -from .foxgay import FoxgayIE from .foxnews import ( - FoxNewsIE, FoxNewsArticleIE, + FoxNewsIE, FoxNewsVideoIE, ) from .foxsports import FoxSportsIE from .fptplay import FptplayIE +from .francaisfacile import FrancaisFacileIE from .franceinter import FranceInterIE from .francetv import ( FranceTVIE, - FranceTVSiteIE, FranceTVInfoIE, + FranceTVSiteIE, ) from .freesound import FreesoundIE from .freespeech import FreespeechIE -from .frontendmasters import ( - FrontendMastersIE, - FrontendMastersLessonIE, - FrontendMastersCourseIE -) from .freetv import ( FreeTvIE, FreeTvMoviesIE, ) -from .fujitv import FujiTVFODPlus7IE -from .funimation import ( - FunimationIE, - FunimationPageIE, - FunimationShowIE, +from .frontendmasters import ( + FrontendMastersCourseIE, + FrontendMastersIE, + FrontendMastersLessonIE, ) +from .fujitv import FujiTVFODPlus7IE from .funk import FunkIE from .funker530 import Funker530IE -from .fusion import FusionIE from .fuyintv import FuyinTVIE from .gab import ( - GabTVIE, GabIE, + GabTVIE, ) from .gaia import GaiaIE -from .gameinformer import GameInformerIE +from .gamedevtv import GameDevTVDashboardIE from .gamejolt import ( - GameJoltIE, - GameJoltUserIE, + GameJoltCommunityIE, GameJoltGameIE, GameJoltGameSoundtrackIE, - GameJoltCommunityIE, + GameJoltIE, GameJoltSearchIE, + GameJoltUserIE, ) from .gamespot import GameSpotIE from .gamestar import GameStarIE from .gaskrank import GaskrankIE from .gazeta import GazetaIE +from .gbnews import GBNewsIE from .gdcvault import GDCVaultIE from .gedidigital import GediDigitalIE from .generic import GenericIE +from .genericembeds import ( + HTML5MediaEmbedIE, + QuotedHTMLIE, +) from .genius import ( GeniusIE, GeniusLyricsIE, ) +from .germanupa import GermanupaIE +from .getcourseru import ( + GetCourseRuIE, + GetCourseRuPlayerIE, +) from .gettr import ( GettrIE, GettrStreamingIE, ) -from .gfycat import GfycatIE from .giantbomb import GiantBombIE -from .giga import GigaIE from .glide import GlideIE from .globalplayer import ( + GlobalPlayerAudioEpisodeIE, + GlobalPlayerAudioIE, GlobalPlayerLiveIE, GlobalPlayerLivePlaylistIE, - GlobalPlayerAudioIE, - GlobalPlayerAudioEpisodeIE, - GlobalPlayerVideoIE + GlobalPlayerVideoIE, ) from .globo import ( - GloboIE, GloboArticleIE, + GloboIE, +) +from .glomex import ( + GlomexEmbedIE, + GlomexIE, ) from .gmanetwork import GMANetworkVideoIE from .go import GoIE +from .godresource import GodResourceIE from .godtube import GodTubeIE from .gofile import GofileIE from .golem import GolemIE from .goodgame import GoodGameIE from .googledrive import ( - GoogleDriveIE, GoogleDriveFolderIE, + GoogleDriveIE, ) from .googlepodcasts import ( - GooglePodcastsIE, GooglePodcastsFeedIE, + GooglePodcastsIE, ) from .googlesearch import GoogleSearchIE -from .gopro import GoProIE from .goplay import GoPlayIE +from .gopro import GoProIE from .goshgay import GoshgayIE from .gotostage import GoToStageIE from .gputechconf import GPUTechConfIE +from .graspop import GraspopIE from .gronkh import ( - GronkhIE, GronkhFeedIE, - GronkhVodsIE + GronkhIE, + GronkhVodsIE, ) from .groupon import GrouponIE from .harpodeon import HarpodeonIE @@ -745,13 +794,11 @@ from .hbo import HBOIE from .hearthisat import HearThisAtIE from .heise import HeiseIE from .hellporno import HellPornoIE -from .helsinki import HelsinkiIE from .hgtv import HGTVComShowIE -from .hketv import HKETVIE from .hidive import HiDiveIE from .historicfilms import HistoricFilmsIE -from .hitbox import HitboxIE, HitboxLiveIE from .hitrecord import HitRecordIE +from .hketv import HKETVIE from .hollywoodreporter import ( HollywoodReporterIE, HollywoodReporterPlaylistIE, @@ -760,13 +807,11 @@ from .holodex import HolodexIE from .hotnewhiphop import HotNewHipHopIE from .hotstar import ( HotStarIE, - HotStarPrefixIE, HotStarPlaylistIE, + HotStarPrefixIE, HotStarSeasonIE, HotStarSeriesIE, ) -from .howcast import HowcastIE -from .howstuffworks import HowStuffWorksIE from .hrefli import HrefLiRedirectIE from .hrfensehen import HRFernsehenIE from .hrti import ( @@ -774,48 +819,49 @@ from .hrti import ( HRTiPlaylistIE, ) from .hse import ( - HSEShowIE, HSEProductIE, -) -from .genericembeds import ( - HTML5MediaEmbedIE, - QuotedHTMLIE, + HSEShowIE, ) from .huajiao import HuajiaoIE -from .huya import HuyaLiveIE from .huffpost import HuffPostIE from .hungama import ( + HungamaAlbumPlaylistIE, HungamaIE, HungamaSongIE, - HungamaAlbumPlaylistIE, +) +from .huya import ( + HuyaLiveIE, + HuyaVideoIE, ) from .hypem import HypemIE from .hypergryph import MonsterSirenHypergryphMusicIE from .hytale import HytaleIE from .icareus import IcareusIE from .ichinanalive import ( - IchinanaLiveIE, IchinanaLiveClipIE, + IchinanaLiveIE, + IchinanaLiveVODIE, ) from .idolplus import IdolPlusIE from .ign import ( IGNIE, - IGNVideoIE, IGNArticleIE, + IGNVideoIE, ) from .iheart import ( IHeartRadioIE, IHeartRadioPodcastIE, ) +from .ilpost import IlPostIE from .iltalehti import IltalehtiIE from .imdb import ( ImdbIE, - ImdbListIE + ImdbListIE, ) from .imgur import ( - ImgurIE, ImgurAlbumIE, ImgurGalleryIE, + ImgurIE, ) from .ina import InaIE from .inc import IncIE @@ -824,20 +870,20 @@ from .infoq import InfoQIE from .instagram import ( InstagramIE, InstagramIOSIE, - InstagramUserIE, - InstagramTagIE, InstagramStoryIE, + InstagramTagIE, + InstagramUserIE, ) from .internazionale import InternazionaleIE from .internetvideoarchive import InternetVideoArchiveIE from .iprima import ( + IPrimaCNNIE, IPrimaIE, - IPrimaCNNIE ) from .iqiyi import ( - IqiyiIE, + IqAlbumIE, IqIE, - IqAlbumIE + IqiyiIE, ) from .islamchannel import ( IslamChannelIE, @@ -845,18 +891,19 @@ from .islamchannel import ( ) from .israelnationalnews import IsraelNationalNewsIE from .itprotv import ( + ITProTVCourseIE, ITProTVIE, - ITProTVCourseIE ) from .itv import ( - ITVIE, ITVBTCCIE, + ITVIE, ) from .ivi import ( + IviCompilationIE, IviIE, - IviCompilationIE ) from .ivideon import IvideonIE +from .ivoox import IvooxIE from .iwara import ( IwaraIE, IwaraPlaylistIE, @@ -864,92 +911,108 @@ from .iwara import ( ) from .ixigua import IxiguaIE from .izlesene import IzleseneIE -from .jable import ( - JableIE, - JablePlaylistIE, -) from .jamendo import ( - JamendoIE, JamendoAlbumIE, + JamendoIE, ) from .japandiet import ( + SangiinIE, + SangiinInstructionIE, ShugiinItvLiveIE, ShugiinItvLiveRoomIE, ShugiinItvVodIE, - SangiinInstructionIE, - SangiinIE, ) from .jeuxvideo import JeuxVideoIE -from .jove import JoveIE +from .jiocinema import ( + JioCinemaIE, + JioCinemaSeriesIE, +) +from .jiosaavn import ( + JioSaavnAlbumIE, + JioSaavnPlaylistIE, + JioSaavnSongIE, +) from .joj import JojIE +from .joqrag import JoqrAgIE +from .jove import JoveIE from .jstream import JStreamIE +from .jtbc import ( + JTBCIE, + JTBCProgramIE, +) from .jwplatform import JWPlatformIE from .kakao import KakaoIE from .kaltura import KalturaIE -from .kanal2 import Kanal2IE from .kankanews import KankaNewsIE from .karaoketv import KaraoketvIE -from .karrierevideos import KarriereVideosIE -from .keezmovies import KeezMoviesIE from .kelbyone import KelbyOneIE +from .kenh14 import ( + Kenh14PlaylistIE, + Kenh14VideoIE, +) from .khanacademy import ( KhanAcademyIE, KhanAcademyUnitIE, ) from .kick import ( + KickClipIE, KickIE, KickVODIE, ) from .kicker import KickerIE from .kickstarter import KickStarterIE +from .kika import ( + KikaIE, + KikaPlaylistIE, +) from .kinja import KinjaEmbedIE from .kinopoisk import KinoPoiskIE from .kommunetv import KommunetvIE from .kompas import KompasVideoIE -from .konserthusetplay import KonserthusetPlayIE from .koo import KooIE -from .kth import KTHIE from .krasview import KrasViewIE +from .kth import KTHIE from .ku6 import Ku6IE -from .kusi import KUSIIE +from .kukululive import KukuluLiveIE from .kuwo import ( - KuwoIE, KuwoAlbumIE, - KuwoChartIE, - KuwoSingerIE, KuwoCategoryIE, + KuwoChartIE, + KuwoIE, KuwoMvIE, + KuwoSingerIE, ) from .la7 import ( LA7IE, LA7PodcastEpisodeIE, LA7PodcastIE, ) -from .laola1tv import ( - Laola1TvEmbedIE, - Laola1TvIE, - EHFTVIE, - ITTFIE, +from .laracasts import ( + LaracastsIE, + LaracastsPlaylistIE, ) from .lastfm import ( LastFMIE, LastFMPlaylistIE, LastFMUserIE, ) +from .laxarxames import LaXarxaMesIE from .lbry import ( LBRYIE, LBRYChannelIE, + LBRYPlaylistIE, ) from .lci import LCIIE from .lcp import ( - LcpPlayIE, LcpIE, + LcpPlayIE, ) +from .learningonscreen import LearningOnScreenIE from .lecture2go import Lecture2GoIE from .lecturio import ( - LecturioIE, LecturioCourseIE, LecturioDeCourseIE, + LecturioIE, ) from .leeco import ( LeIE, @@ -966,24 +1029,24 @@ from .lenta import LentaIE from .libraryofcongress import LibraryOfCongressIE from .libsyn import LibsynIE from .lifenews import ( - LifeNewsIE, LifeEmbedIE, + LifeNewsIE, ) from .likee import ( LikeeIE, - LikeeUserIE + LikeeUserIE, ) from .limelight import ( - LimelightMediaIE, LimelightChannelIE, LimelightChannelListIE, + LimelightMediaIE, ) from .linkedin import ( + LinkedInEventsIE, LinkedInIE, - LinkedInLearningIE, LinkedInLearningCourseIE, + LinkedInLearningIE, ) -from .linuxacademy import LinuxAcademyIE from .liputan6 import Liputan6IE from .listennotes import ListenNotesIE from .litv import LiTVIE @@ -994,41 +1057,45 @@ from .livestream import ( LivestreamShortenerIE, ) from .livestreamfails import LivestreamfailsIE -from .lnkgo import ( - LnkGoIE, - LnkIE, +from .lnk import LnkIE +from .loco import LocoIE +from .loom import ( + LoomFolderIE, + LoomIE, ) -from .localnews8 import LocalNews8IE from .lovehomeporn import LoveHomePornIE from .lrt import ( LRTVODIE, - LRTStreamIE + LRTRadioIE, + LRTStreamIE, ) -from .lumni import ( - LumniIE +from .lsm import ( + LSMLREmbedIE, + LSMLTVEmbedIE, + LSMReplayIE, ) +from .lumni import LumniIE from .lynda import ( + LyndaCourseIE, LyndaIE, - LyndaCourseIE ) -from .m6 import M6IE +from .maariv import MaarivIE from .magellantv import MagellanTVIE -from .magentamusik360 import MagentaMusik360IE +from .magentamusik import MagentaMusikIE from .mailru import ( MailRuIE, MailRuMusicIE, MailRuMusicSearchIE, ) from .mainstreaming import MainStreamingIE -from .malltv import MallTVIE from .mangomolo import ( - MangomoloVideoIE, MangomoloLiveIE, + MangomoloVideoIE, ) from .manoto import ( ManotoTVIE, - ManotoTVShowIE, ManotoTVLiveIE, + ManotoTVShowIE, ) from .manyvids import ManyVidsIE from .maoritv import MaoriTVIE @@ -1039,17 +1106,19 @@ from .markiza import ( from .massengeschmacktv import MassengeschmackTVIE from .masters import MastersIE from .matchtv import MatchTVIE +from .mbn import MBNIE from .mdr import MDRIE from .medaltv import MedalTVIE from .mediaite import MediaiteIE from .mediaklikk import MediaKlikkIE +from .medialaan import MedialaanIE from .mediaset import ( MediasetIE, MediasetShowIE, ) from .mediasite import ( - MediasiteIE, MediasiteCatalogIE, + MediasiteIE, MediasiteNamedCatalogIE, ) from .mediastream import ( @@ -1059,109 +1128,108 @@ from .mediastream import ( from .mediaworksnz import MediaWorksNZVODIE from .medici import MediciIE from .megaphone import MegaphoneIE +from .megatvcom import ( + MegaTVComEmbedIE, + MegaTVComIE, +) from .meipai import MeipaiIE from .melonvod import MelonVODIE -from .meta import METAIE -from .metacafe import MetacafeIE from .metacritic import MetacriticIE -from .mgoon import MgoonIE from .mgtv import MGTVIE -from .miaopai import MiaoPaiIE +from .microsoftembed import ( + MicrosoftBuildIE, + MicrosoftEmbedIE, + MicrosoftLearnEpisodeIE, + MicrosoftLearnPlaylistIE, + MicrosoftLearnSessionIE, + MicrosoftMediusIE, +) from .microsoftstream import MicrosoftStreamIE -from .microsoftvirtualacademy import ( - MicrosoftVirtualAcademyIE, - MicrosoftVirtualAcademyCourseIE, -) -from .microsoftembed import MicrosoftEmbedIE -from .mildom import ( - MildomIE, - MildomVodIE, - MildomClipIE, - MildomUserVodIE, -) from .minds import ( - MindsIE, MindsChannelIE, MindsGroupIE, + MindsIE, ) -from .ministrygrid import MinistryGridIE from .minoto import MinotoIE -from .miomio import MioMioIE from .mirrativ import ( MirrativIE, MirrativUserIE, ) from .mirrorcouk import MirrorCoUKIE -from .mit import TechTVMITIE, OCWMITIE +from .mit import ( + OCWMITIE, + TechTVMITIE, +) from .mitele import MiTeleIE from .mixch import ( - MixchIE, MixchArchiveIE, + MixchIE, + MixchMovieIE, ) from .mixcloud import ( MixcloudIE, - MixcloudUserIE, MixcloudPlaylistIE, + MixcloudUserIE, ) from .mlb import ( MLBIE, - MLBVideoIE, MLBTVIE, MLBArticleIE, + MLBVideoIE, ) from .mlssoccer import MLSSoccerIE -from .mnet import MnetIE from .mocha import MochaVideoIE -from .moevideo import MoeVideoIE -from .mofosex import ( - MofosexIE, - MofosexEmbedIE, -) +from .mojevideo import MojevideoIE from .mojvideo import MojvideoIE -from .morningstar import MorningstarIE +from .monstercat import MonstercatIE from .motherless import ( - MotherlessIE, - MotherlessGroupIE, MotherlessGalleryIE, + MotherlessGroupIE, + MotherlessIE, + MotherlessUploaderIE, ) from .motorsport import MotorsportIE -from .movieclips import MovieClipsIE from .moviepilot import MoviepilotIE from .moview import MoviewPlayIE from .moviezine import MoviezineIE from .movingimage import MovingImageIE from .msn import MSNIE from .mtv import ( - MTVIE, - MTVVideoIE, - MTVServicesEmbeddedIE, MTVDEIE, - MTVJapanIE, + MTVIE, MTVItaliaIE, MTVItaliaProgrammaIE, + MTVJapanIE, + MTVServicesEmbeddedIE, + MTVVideoIE, ) from .muenchentv import MuenchenTVIE -from .murrtube import MurrtubeIE, MurrtubeUserIE +from .murrtube import ( + MurrtubeIE, + MurrtubeUserIE, +) from .museai import MuseAIIE from .musescore import MuseScoreIE from .musicdex import ( - MusicdexSongIE, MusicdexAlbumIE, MusicdexArtistIE, MusicdexPlaylistIE, + MusicdexSongIE, +) +from .mx3 import ( + Mx3IE, + Mx3NeoIE, + Mx3VolksmusikIE, ) -from .mwave import MwaveIE, MwaveMeetGreetIE from .mxplayer import ( MxplayerIE, MxplayerShowIE, ) -from .mychannels import MyChannelsIE -from .myspace import MySpaceIE, MySpaceAlbumIE -from .myspass import MySpassIE -from .myvi import ( - MyviIE, - MyviEmbedIE, +from .myspace import ( + MySpaceAlbumIE, + MySpaceIE, ) +from .myspass import MySpassIE from .myvideoge import MyVideoGeIE from .myvidster import MyVidsterIE from .mzaalo import MzaaloIE @@ -1174,8 +1242,8 @@ from .nate import ( NateProgramIE, ) from .nationalgeographic import ( - NationalGeographicVideoIE, NationalGeographicTVIE, + NationalGeographicVideoIE, ) from .naver import ( NaverIE, @@ -1183,12 +1251,12 @@ from .naver import ( NaverNowIE, ) from .nba import ( - NBAWatchEmbedIE, - NBAWatchIE, - NBAWatchCollectionIE, - NBAEmbedIE, NBAIE, NBAChannelIE, + NBAEmbedIE, + NBAWatchCollectionIE, + NBAWatchEmbedIE, + NBAWatchIE, ) from .nbc import ( NBCIE, @@ -1202,53 +1270,60 @@ from .nbc import ( ) from .ndr import ( NDRIE, - NJoyIE, NDREmbedBaseIE, NDREmbedIE, NJoyEmbedIE, + NJoyIE, ) from .ndtv import NDTVIE from .nebula import ( + NebulaChannelIE, + NebulaClassIE, NebulaIE, NebulaSubscriptionsIE, - NebulaChannelIE, ) from .nekohacker import NekoHackerIE from .nerdcubed import NerdCubedFeedIE -from .netzkino import NetzkinoIE +from .nest import ( + NestClipIE, + NestIE, +) from .neteasemusic import ( - NetEaseMusicIE, NetEaseMusicAlbumIE, - NetEaseMusicSingerIE, + NetEaseMusicDjRadioIE, + NetEaseMusicIE, NetEaseMusicListIE, NetEaseMusicMvIE, NetEaseMusicProgramIE, - NetEaseMusicDjRadioIE, + NetEaseMusicSingerIE, ) from .netverse import ( NetverseIE, NetversePlaylistIE, NetverseSearchIE, ) +from .netzkino import NetzkinoIE from .newgrounds import ( NewgroundsIE, NewgroundsPlaylistIE, NewgroundsUserIE, ) from .newspicks import NewsPicksIE -from .newstube import NewstubeIE from .newsy import NewsyIE from .nextmedia import ( - NextMediaIE, - NextMediaActionNewsIE, AppleDailyIE, + NextMediaActionNewsIE, + NextMediaIE, NextTVIE, ) from .nexx import ( - NexxIE, NexxEmbedIE, + NexxIE, +) +from .nfb import ( + NFBIE, + NFBSeriesIE, ) -from .nfb import NFBIE from .nfhsnetwork import NFHSNetworkIE from .nfl import ( NFLIE, @@ -1257,51 +1332,54 @@ from .nfl import ( NFLPlusReplayIE, ) from .nhk import ( - NhkVodIE, - NhkVodProgramIE, NhkForSchoolBangumiIE, - NhkForSchoolSubjectIE, NhkForSchoolProgramListIE, + NhkForSchoolSubjectIE, NhkRadioNewsPageIE, NhkRadiruIE, NhkRadiruLiveIE, + NhkVodIE, + NhkVodProgramIE, ) from .nhl import NHLIE from .nick import ( - NickIE, NickBrIE, NickDeIE, - NickNightIE, + NickIE, NickRuIE, ) from .niconico import ( - NiconicoIE, - NiconicoPlaylistIE, - NiconicoUserIE, - NiconicoSeriesIE, NiconicoHistoryIE, + NiconicoIE, + NiconicoLiveIE, + NiconicoPlaylistIE, + NiconicoSeriesIE, + NiconicoUserIE, NicovideoSearchDateIE, NicovideoSearchIE, NicovideoSearchURLIE, NicovideoTagURLIE, - NiconicoLiveIE, ) +from .niconicochannelplus import ( + NiconicoChannelPlusChannelLivesIE, + NiconicoChannelPlusChannelVideosIE, + NiconicoChannelPlusIE, +) +from .ninaprotocol import NinaProtocolIE from .ninecninemedia import ( - NineCNineMediaIE, CPTwentyFourIE, + NineCNineMediaIE, ) from .ninegag import NineGagIE +from .ninenews import NineNewsIE from .ninenow import NineNowIE from .nintendo import NintendoIE from .nitter import NitterIE -from .njpwworld import NJPWWorldIE from .nobelprize import NobelPrizeIE from .noice import NoicePodcastIE from .nonktube import NonkTubeIE from .noodlemagazine import NoodleMagazineIE from .noovo import NoovoIE -from .normalboots import NormalbootsIE -from .nosvideo import NosVideoIE from .nosnl import NOSNLArticleIE from .nova import ( NovaEmbedIE, @@ -1315,100 +1393,105 @@ from .nowness import ( ) from .noz import NozIE from .npo import ( - AndereTijdenIE, NPOIE, - NPOLiveIE, - NPORadioIE, - NPORadioFragmentIE, - SchoolTVIE, - HetKlokhuisIE, VPROIE, WNLIE, + AndereTijdenIE, + HetKlokhuisIE, + NPOLiveIE, + NPORadioFragmentIE, + NPORadioIE, + SchoolTVIE, ) from .npr import NprIE from .nrk import ( NRKIE, - NRKPlaylistIE, - NRKSkoleIE, NRKTVIE, - NRKTVDirekteIE, + NRKPlaylistIE, NRKRadioPodkastIE, + NRKSkoleIE, + NRKTVDirekteIE, NRKTVEpisodeIE, NRKTVEpisodesIE, NRKTVSeasonIE, NRKTVSeriesIE, ) from .nrl import NRLTVIE +from .nts import NTSLiveIE from .ntvcojp import NTVCoJpCUIE from .ntvde import NTVDeIE from .ntvru import NTVRuIE from .nubilesporn import NubilesPornIE -from .nytimes import ( - NYTimesIE, - NYTimesArticleIE, - NYTimesCookingIE, +from .nuum import ( + NuumLiveIE, + NuumMediaIE, + NuumTabIE, ) from .nuvid import NuvidIE +from .nytimes import ( + NYTimesArticleIE, + NYTimesCookingIE, + NYTimesCookingRecipeIE, + NYTimesIE, +) from .nzherald import NZHeraldIE from .nzonscreen import NZOnScreenIE from .nzz import NZZIE -from .odatv import OdaTVIE from .odkmedia import OnDemandChinaEpisodeIE from .odnoklassniki import OdnoklassnikiIE from .oftv import ( OfTVIE, - OfTVPlaylistIE + OfTVPlaylistIE, ) from .oktoberfesttv import OktoberfestTVIE from .olympics import OlympicsReplayIE from .on24 import On24IE -from .ondemandkorea import OnDemandKoreaIE +from .ondemandkorea import ( + OnDemandKoreaIE, + OnDemandKoreaProgramIE, +) from .onefootball import OneFootballIE from .onenewsnz import OneNewsNZIE from .oneplace import OnePlacePodcastIE from .onet import ( - OnetIE, OnetChannelIE, + OnetIE, OnetMVPIE, OnetPlIE, ) from .onionstudios import OnionStudiosIE -from .ooyala import ( - OoyalaIE, - OoyalaExternalIE, -) from .opencast import ( OpencastIE, OpencastPlaylistIE, ) from .openrec import ( - OpenRecIE, OpenRecCaptureIE, + OpenRecIE, OpenRecMovieIE, ) from .ora import OraTVIE from .orf import ( - ORFTVthekIE, - ORFFM4StoryIE, - ORFRadioIE, ORFIPTVIE, + ORFONIE, + ORFFM4StoryIE, + ORFPodcastIE, + ORFRadioIE, ) from .outsidetv import OutsideTVIE from .owncloud import OwnCloudIE from .packtpub import ( - PacktPubIE, PacktPubCourseIE, + PacktPubIE, ) from .palcomp3 import ( - PalcoMP3IE, PalcoMP3ArtistIE, + PalcoMP3IE, PalcoMP3VideoIE, ) -from .pandoratv import PandoraTVIE from .panopto import ( PanoptoIE, PanoptoListIE, - PanoptoPlaylistIE + PanoptoPlaylistIE, ) from .paramountplus import ( ParamountPlusIE, @@ -1416,13 +1499,23 @@ from .paramountplus import ( ) from .parler import ParlerIE from .parlview import ParlviewIE -from .patreon import ( - PatreonIE, - PatreonCampaignIE +from .parti import ( + PartiLivestreamIE, + PartiVideoIE, +) +from .patreon import ( + PatreonCampaignIE, + PatreonIE, +) +from .pbs import ( + PBSIE, + PBSKidsIE, ) -from .pbs import PBSIE, PBSKidsIE from .pearvideo import PearVideoIE -from .peekvids import PeekVidsIE, PlayVidsIE +from .peekvids import ( + PeekVidsIE, + PlayVidsIE, +) from .peertube import ( PeerTubeIE, PeerTubePlaylistIE, @@ -1430,9 +1523,8 @@ from .peertube import ( from .peertv import PeerTVIE from .peloton import ( PelotonIE, - PelotonLiveIE + PelotonLiveIE, ) -from .people import PeopleIE from .performgroup import PerformGroupIE from .periscope import ( PeriscopeIE, @@ -1442,6 +1534,7 @@ from .pgatour import PGATourIE from .philharmoniedeparis import PhilharmonieDeParisIE from .phoenix import PhoenixIE from .photobucket import PhotobucketIE +from .pialive import PiaLiveIE from .piapro import PiaproIE from .picarto import ( PicartoIE, @@ -1450,8 +1543,12 @@ from .picarto import ( from .piksel import PikselIE from .pinkbike import PinkbikeIE from .pinterest import ( - PinterestIE, PinterestCollectionIE, + PinterestIE, +) +from .piramidetv import ( + PiramideTVChannelIE, + PiramideTVIE, ) from .pixivsketch import ( PixivSketchIE, @@ -1460,131 +1557,142 @@ from .pixivsketch import ( from .pladform import PladformIE from .planetmarathi import PlanetMarathiIE from .platzi import ( - PlatziIE, PlatziCourseIE, + PlatziIE, ) -from .playfm import PlayFMIE from .playplustv import PlayPlusTVIE -from .plays import PlaysTVIE -from .playstuff import PlayStuffIE from .playsuisse import PlaySuisseIE from .playtvak import PlaytvakIE -from .playvid import PlayvidIE from .playwire import PlaywireIE -from .plutotv import PlutoTVIE from .pluralsight import ( - PluralsightIE, PluralsightCourseIE, + PluralsightIE, +) +from .plutotv import PlutoTVIE +from .plvideo import PlVideoIE +from .podbayfm import ( + PodbayFMChannelIE, + PodbayFMIE, ) -from .podbayfm import PodbayFMIE, PodbayFMChannelIE from .podchaser import PodchaserIE from .podomatic import PodomaticIE -from .pokemon import ( - PokemonIE, - PokemonWatchIE, -) from .pokergo import ( - PokerGoIE, PokerGoCollectionIE, + PokerGoIE, ) from .polsatgo import PolsatGoIE from .polskieradio import ( - PolskieRadioIE, - PolskieRadioLegacyIE, PolskieRadioAuditionIE, PolskieRadioCategoryIE, + PolskieRadioIE, + PolskieRadioLegacyIE, PolskieRadioPlayerIE, PolskieRadioPodcastIE, PolskieRadioPodcastListIE, ) from .popcorntimes import PopcorntimesIE from .popcorntv import PopcornTVIE -from .porn91 import Porn91IE -from .porncom import PornComIE +from .pornbox import PornboxIE from .pornflip import PornFlipIE -from .pornhd import PornHdIE from .pornhub import ( PornHubIE, - PornHubUserIE, - PornHubPlaylistIE, PornHubPagedVideoListIE, + PornHubPlaylistIE, + PornHubUserIE, PornHubUserVideosUploadIE, ) from .pornotube import PornotubeIE from .pornovoisines import PornoVoisinesIE from .pornoxo import PornoXOIE -from .pornez import PornezIE -from .puhutv import ( - PuhuTVIE, - PuhuTVSerieIE, +from .pr0gramm import Pr0grammIE +from .prankcast import ( + PrankCastIE, + PrankCastPostIE, ) -from .pr0gramm import Pr0grammStaticIE, Pr0grammIE -from .prankcast import PrankCastIE from .premiershiprugby import PremiershipRugbyIE from .presstv import PressTVIE from .projectveritas import ProjectVeritasIE from .prosiebensat1 import ProSiebenSat1IE from .prx import ( - PRXStoryIE, - PRXSeriesIE, PRXAccountIE, + PRXSeriesIE, + PRXSeriesSearchIE, PRXStoriesSearchIE, - PRXSeriesSearchIE + PRXStoryIE, +) +from .puhutv import ( + PuhuTVIE, + PuhuTVSerieIE, ) from .puls4 import Puls4IE from .pyvideo import PyvideoIE from .qdance import QDanceIE from .qingting import QingTingIE from .qqmusic import ( - QQMusicIE, - QQMusicSingerIE, QQMusicAlbumIE, - QQMusicToplistIE, + QQMusicIE, QQMusicPlaylistIE, + QQMusicSingerIE, + QQMusicToplistIE, + QQMusicVideoIE, ) from .r7 import ( R7IE, R7ArticleIE, ) -from .radiko import RadikoIE, RadikoRadioIE +from .radiko import ( + RadikoIE, + RadikoRadioIE, +) from .radiocanada import ( - RadioCanadaIE, RadioCanadaAudioVideoIE, + RadioCanadaIE, +) +from .radiocomercial import ( + RadioComercialIE, + RadioComercialPlaylistIE, ) from .radiode import RadioDeIE +from .radiofrance import ( + FranceCultureIE, + RadioFranceIE, + RadioFranceLiveIE, + RadioFrancePodcastIE, + RadioFranceProfileIE, + RadioFranceProgramScheduleIE, +) from .radiojavan import RadioJavanIE -from .radiobremen import RadioBremenIE -from .radiofrance import FranceCultureIE, RadioFranceIE -from .radiozet import RadioZetPodcastIE from .radiokapital import ( RadioKapitalIE, RadioKapitalShowIE, ) +from .radioradicale import RadioRadicaleIE +from .radiozet import RadioZetPodcastIE from .radlive import ( - RadLiveIE, RadLiveChannelIE, + RadLiveIE, RadLiveSeasonIE, ) from .rai import ( - RaiIE, RaiCulturaIE, + RaiIE, + RaiNewsIE, RaiPlayIE, RaiPlayLiveIE, RaiPlayPlaylistIE, RaiPlaySoundIE, RaiPlaySoundLiveIE, RaiPlaySoundPlaylistIE, - RaiNewsIE, RaiSudtirolIE, ) from .raywenderlich import ( - RayWenderlichIE, RayWenderlichCourseIE, + RayWenderlichIE, ) -from .rbmaradio import RBMARadioIE from .rbgtum import ( - RbgTumIE, RbgTumCourseIE, + RbgTumIE, + RbgTumNewCourseIE, ) from .rcs import ( RCSIE, @@ -1597,22 +1705,24 @@ from .rcti import ( RCTIPlusTVIE, ) from .rds import RDSIE -from .recurbate import RecurbateIE -from .redbee import ParliamentLiveUKIE, RTBFIE +from .redbee import ( + RTBFIE, + ParliamentLiveUKIE, +) from .redbulltv import ( - RedBullTVIE, RedBullEmbedIE, - RedBullTVRrnContentIE, RedBullIE, + RedBullTVIE, + RedBullTVRrnContentIE, ) from .reddit import RedditIE +from .redge import RedCDNLivxIE from .redgifs import ( RedGifsIE, RedGifsSearchIE, RedGifsUserIE, ) from .redtube import RedTubeIE -from .regiotv import RegioTVIE from .rentv import ( RENTVIE, RENTVArticleIE, @@ -1621,176 +1731,187 @@ from .restudy import RestudyIE from .reuters import ReutersIE from .reverbnation import ReverbNationIE from .rheinmaintv import RheinMainTVIE -from .rice import RICEIE +from .ridehome import RideHomeIE +from .rinsefm import ( + RinseFMArtistPlaylistIE, + RinseFMIE, +) from .rmcdecouverte import RMCDecouverteIE from .rockstargames import RockstarGamesIE from .rokfin import ( - RokfinIE, - RokfinStackIE, RokfinChannelIE, + RokfinIE, RokfinSearchIE, + RokfinStackIE, +) +from .roosterteeth import ( + RoosterTeethIE, + RoosterTeethSeriesIE, ) -from .roosterteeth import RoosterTeethIE, RoosterTeethSeriesIE from .rottentomatoes import RottenTomatoesIE +from .roya import RoyaLiveIE from .rozhlas import ( + MujRozhlasIE, RozhlasIE, RozhlasVltavaIE, - MujRozhlasIE, ) -from .rte import RteIE, RteRadioIE +from .rte import ( + RteIE, + RteRadioIE, +) +from .rtl2 import RTL2IE from .rtlnl import ( - RtlNlIE, - RTLLuTeleVODIE, RTLLuArticleIE, RTLLuLiveIE, RTLLuRadioIE, -) -from .rtl2 import ( - RTL2IE, - RTL2YouIE, - RTL2YouSeriesIE, + RTLLuTeleVODIE, + RtlNlIE, ) from .rtnews import ( - RTNewsIE, RTDocumentryIE, RTDocumentryPlaylistIE, + RTNewsIE, RuptlyIE, ) from .rtp import RTPIE from .rtrfm import RTRFMIE from .rts import RTSIE from .rtvcplay import ( - RTVCPlayIE, - RTVCPlayEmbedIE, RTVCKalturaIE, + RTVCPlayEmbedIE, + RTVCPlayIE, ) from .rtve import ( RTVEALaCartaIE, RTVEAudioIE, RTVELiveIE, - RTVEInfantilIE, RTVETelevisionIE, ) -from .rtvnh import RTVNHIE from .rtvs import RTVSIE -from .rtvslo import RTVSLOIE -from .ruhd import RUHDIE +from .rtvslo import ( + RTVSLOIE, + RTVSLOShowIE, +) +from .rudovideo import RudoVideoIE from .rule34video import Rule34VideoIE from .rumble import ( + RumbleChannelIE, RumbleEmbedIE, RumbleIE, - RumbleChannelIE, ) from .rutube import ( - RutubeIE, RutubeChannelIE, RutubeEmbedIE, + RutubeIE, RutubeMovieIE, RutubePersonIE, RutubePlaylistIE, RutubeTagsIE, ) -from .glomex import ( - GlomexIE, - GlomexEmbedIE, -) -from .megatvcom import ( - MegaTVComIE, - MegaTVComEmbedIE, -) -from .ant1newsgr import ( - Ant1NewsGrWatchIE, - Ant1NewsGrArticleIE, - Ant1NewsGrEmbedIE, -) from .rutv import RUTVIE from .ruutu import RuutuIE from .ruv import ( RuvIE, - RuvSpilaIE + RuvSpilaIE, +) +from .s4c import ( + S4CIE, + S4CSeriesIE, ) -from .s4c import S4CIE from .safari import ( - SafariIE, SafariApiIE, SafariCourseIE, + SafariIE, ) from .saitosan import SaitosanIE from .samplefocus import SampleFocusIE from .sapo import SapoIE -from .savefrom import SaveFromIE from .sbs import SBSIE +from .sbscokr import ( + SBSCoKrAllvodProgramIE, + SBSCoKrIE, + SBSCoKrProgramsVodIE, +) from .screen9 import Screen9IE from .screencast import ScreencastIE from .screencastify import ScreencastifyIE from .screencastomatic import ScreencastOMaticIE +from .screenrec import ScreenRecIE from .scrippsnetworks import ( - ScrippsNetworksWatchIE, ScrippsNetworksIE, + ScrippsNetworksWatchIE, ) +from .scrolller import ScrolllerIE from .scte import ( SCTEIE, SCTECourseIE, ) -from .scrolller import ScrolllerIE -from .seeker import SeekerIE +from .sejmpl import SejmIE +from .sen import SenIE from .senalcolombia import SenalColombiaLiveIE -from .senategov import SenateISVPIE, SenateGovIE +from .senategov import ( + SenateGovIE, + SenateISVPIE, +) from .sendtonews import SendtoNewsIE from .servus import ServusIE from .sevenplus import SevenPlusIE from .sexu import SexuIE from .seznamzpravy import ( - SeznamZpravyIE, SeznamZpravyArticleIE, + SeznamZpravyIE, ) from .shahid import ( ShahidIE, ShahidShowIE, ) -from .shared import ( - SharedIE, - VivoIE, -) +from .sharepoint import SharePointIE from .sharevideos import ShareVideosEmbedIE -from .sibnet import SibnetEmbedIE from .shemaroome import ShemarooMeIE from .showroomlive import ShowRoomLiveIE +from .sibnet import SibnetEmbedIE from .simplecast import ( - SimplecastIE, SimplecastEpisodeIE, + SimplecastIE, SimplecastPodcastIE, ) from .sina import SinaIE from .sixplay import SixPlayIE from .skeb import SkebIE -from .skyit import ( - SkyItPlayerIE, - SkyItVideoIE, - SkyItVideoLiveIE, - SkyItIE, - SkyItArteIE, - CieloTVItIE, - TV8ItIE, -) -from .skylinewebcams import SkylineWebcamsIE -from .skynewsarabia import ( - SkyNewsArabiaIE, - SkyNewsArabiaArticleIE, -) -from .skynewsau import SkyNewsAUIE from .sky import ( SkyNewsIE, SkyNewsStoryIE, SkySportsIE, SkySportsNewsIE, ) +from .skyit import ( + CieloTVItIE, + SkyItArteIE, + SkyItIE, + SkyItPlayerIE, + SkyItVideoIE, + SkyItVideoLiveIE, + TV8ItIE, + TV8ItLiveIE, + TV8ItPlaylistIE, +) +from .skylinewebcams import SkylineWebcamsIE +from .skynewsarabia import ( + SkyNewsArabiaArticleIE, + SkyNewsArabiaIE, +) +from .skynewsau import SkyNewsAUIE from .slideshare import SlideshareIE from .slideslive import SlidesLiveIE from .slutload import SlutloadIE from .smotrim import SmotrimIE +from .snapchat import SnapchatSpotlightIE from .snotr import SnotrIE -from .sohu import SohuIE +from .softwhiteunderbelly import SoftWhiteUnderbellyIE +from .sohu import ( + SohuIE, + SohuVIE, +) from .sonyliv import ( SonyLIVIE, SonyLIVSeriesIE, @@ -1798,46 +1919,39 @@ from .sonyliv import ( from .soundcloud import ( SoundcloudEmbedIE, SoundcloudIE, - SoundcloudSetIE, + SoundcloudPlaylistIE, SoundcloudRelatedIE, + SoundcloudSearchIE, + SoundcloudSetIE, + SoundcloudTrackStationIE, SoundcloudUserIE, SoundcloudUserPermalinkIE, - SoundcloudTrackStationIE, - SoundcloudPlaylistIE, - SoundcloudSearchIE, ) from .soundgasm import ( SoundgasmIE, - SoundgasmProfileIE + SoundgasmProfileIE, ) from .southpark import ( - SouthParkIE, SouthParkDeIE, SouthParkDkIE, SouthParkEsIE, + SouthParkIE, SouthParkLatIE, - SouthParkNlIE + SouthParkNlIE, ) from .sovietscloset import ( SovietsClosetIE, - SovietsClosetPlaylistIE + SovietsClosetPlaylistIE, ) from .spankbang import ( SpankBangIE, SpankBangPlaylistIE, ) -from .spankwire import SpankwireIE from .spiegel import SpiegelIE from .spike import ( BellatorIE, ParamountNetworkIE, ) -from .stageplus import StagePlusVODConcertIE -from .startrek import StarTrekIE -from .stitcher import ( - StitcherIE, - StitcherShowIE, -) from .sport5 import Sport5IE from .sportbox import SportBoxIE from .sportdeutschland import SportDeutschlandIE @@ -1847,12 +1961,14 @@ from .spotify import ( ) from .spreaker import ( SpreakerIE, - SpreakerPageIE, SpreakerShowIE, - SpreakerShowPageIE, ) from .springboardplatform import SpringboardPlatformIE from .sprout import SproutIE +from .sproutvideo import ( + SproutVideoIE, + VidsIoIE, +) from .srgssr import ( SRGSSRIE, SRGSSRPlayIE, @@ -1861,26 +1977,37 @@ from .srmediathek import SRMediathekIE from .stacommu import ( StacommuLiveIE, StacommuVODIE, + TheaterComplexTownPPVIE, + TheaterComplexTownVODIE, ) +from .stageplus import StagePlusVODConcertIE from .stanfordoc import StanfordOpenClassroomIE +from .startrek import StarTrekIE from .startv import StarTVIE from .steam import ( - SteamIE, SteamCommunityBroadcastIE, + SteamIE, +) +from .stitcher import ( + StitcherIE, + StitcherShowIE, ) from .storyfire import ( StoryFireIE, - StoryFireUserIE, StoryFireSeriesIE, + StoryFireUserIE, ) +from .streaks import StreaksIE from .streamable import StreamableIE -from .streamcloud import StreamcloudIE from .streamcz import StreamCZIE -from .streamff import StreamFFIE from .streetvoice import StreetVoiceIE from .stretchinternet import StretchInternetIE from .stripchat import StripchatIE from .stv import STVPlayerIE +from .subsplash import ( + SubsplashIE, + SubsplashPlaylistIE, +) from .substack import SubstackIE from .sunporno import SunPornoIE from .sverigesradio import ( @@ -1894,17 +2021,26 @@ from .svt import ( SVTSeriesIE, ) from .swearnet import SwearnetEpisodeIE -from .swrmediathek import SWRMediathekIE -from .syvdk import SYVDKIE from .syfy import SyfyIE +from .syvdk import SYVDKIE from .sztvhu import SztvHuIE from .tagesschau import TagesschauIE +from .taptap import ( + TapTapAppIE, + TapTapAppIntlIE, + TapTapMomentIE, + TapTapPostIntlIE, +) from .tass import TassIE from .tbs import TBSIE -from .tdslifeway import TDSLifewayIE +from .tbsjp import ( + TBSJPEpisodeIE, + TBSJPPlaylistIE, + TBSJPProgramIE, +) from .teachable import ( - TeachableIE, TeachableCourseIE, + TeachableIE, ) from .teachertube import ( TeacherTubeIE, @@ -1912,11 +2048,10 @@ from .teachertube import ( ) from .teachingchannel import TeachingChannelIE from .teamcoco import ( - TeamcocoIE, ConanClassicIE, + TeamcocoIE, ) from .teamtreehouse import TeamTreeHouseIE -from .techtalks import TechTalksIE from .ted import ( TedEmbedIE, TedPlaylistIE, @@ -1933,15 +2068,18 @@ from .telegram import TelegramEmbedIE from .telemb import TeleMBIE from .telemundo import TelemundoIE from .telequebec import ( - TeleQuebecIE, - TeleQuebecSquatIE, TeleQuebecEmissionIE, + TeleQuebecIE, TeleQuebecLiveIE, + TeleQuebecSquatIE, TeleQuebecVideoIE, ) from .teletask import TeleTaskIE from .telewebion import TelewebionIE -from .tempo import TempoIE, IVXPlayerIE +from .tempo import ( + IVXPlayerIE, + TempoIE, +) from .tencent import ( IflixEpisodeIE, IflixSeriesIE, @@ -1951,110 +2089,107 @@ from .tencent import ( WeTvSeriesIE, ) from .tennistv import TennisTVIE -from .tenplay import TenPlayIE +from .tenplay import ( + TenPlayIE, + TenPlaySeasonIE, +) from .testurl import TestURLIE from .tf1 import TF1IE from .tfo import TFOIE +from .theguardian import ( + TheGuardianPodcastIE, + TheGuardianPodcastPlaylistIE, +) from .theholetv import TheHoleTvIE from .theintercept import TheInterceptIE from .theplatform import ( - ThePlatformIE, ThePlatformFeedIE, + ThePlatformIE, ) from .thestar import TheStarIE from .thesun import TheSunIE -from .theta import ( - ThetaVideoIE, - ThetaStreamIE, -) from .theweatherchannel import TheWeatherChannelIE from .thisamericanlife import ThisAmericanLifeIE -from .thisav import ThisAVIE from .thisoldhouse import ThisOldHouseIE from .thisvid import ( ThisVidIE, ThisVidMemberIE, ThisVidPlaylistIE, ) +from .threeqsdn import ThreeQSDNIE from .threespeak import ( ThreeSpeakIE, ThreeSpeakUserIE, ) -from .threeqsdn import ThreeQSDNIE from .tiktok import ( - TikTokIE, - TikTokUserIE, - TikTokSoundIE, - TikTokEffectIE, - TikTokTagIE, - TikTokVMIE, - TikTokLiveIE, DouyinIE, + TikTokCollectionIE, + TikTokEffectIE, + TikTokIE, + TikTokLiveIE, + TikTokSoundIE, + TikTokTagIE, + TikTokUserIE, + TikTokVMIE, ) -from .tinypic import TinyPicIE from .tmz import TMZIE from .tnaflix import ( - TNAFlixNetworkEmbedIE, - TNAFlixIE, EMPFlixIE, MovieFapIE, + TNAFlixIE, + TNAFlixNetworkEmbedIE, ) from .toggle import ( - ToggleIE, MeWatchIE, + ToggleIE, ) -from .toggo import ( - ToggoIE, -) -from .tokentube import ( - TokentubeIE, - TokentubeChannelIE -) +from .toggo import ToggoIE from .tonline import TOnlineIE from .toongoggles import ToonGogglesIE from .toutv import TouTvIE -from .toypics import ToypicsUserIE, ToypicsIE +from .toypics import ( + ToypicsIE, + ToypicsUserIE, +) from .traileraddict import TrailerAddictIE from .triller import ( TrillerIE, - TrillerUserIE, TrillerShortIE, + TrillerUserIE, ) -from .trilulilu import TriluliluIE from .trovo import ( + TrovoChannelClipIE, + TrovoChannelVodIE, TrovoIE, TrovoVodIE, - TrovoChannelVodIE, - TrovoChannelClipIE, ) from .trtcocuk import TrtCocukVideoIE +from .trtworld import TrtWorldIE from .trueid import TrueIDIE from .trunews import TruNewsIE from .truth import TruthIE from .trutv import TruTVIE from .tube8 import Tube8IE -from .tubetugraz import TubeTuGrazIE, TubeTuGrazSeriesIE +from .tubetugraz import ( + TubeTuGrazIE, + TubeTuGrazSeriesIE, +) from .tubitv import ( TubiTvIE, TubiTvShowIE, ) from .tumblr import TumblrIE from .tunein import ( - TuneInStationIE, - TuneInPodcastIE, TuneInPodcastEpisodeIE, + TuneInPodcastIE, TuneInShortenerIE, + TuneInStationIE, ) -from .tunepk import TunePkIE -from .turbo import TurboIE from .tv2 import ( TV2IE, - TV2ArticleIE, KatsomoIE, MTVUutisetArticleIE, -) -from .tv24ua import ( - TV24UAVideoIE, + TV2ArticleIE, ) from .tv2dk import ( TV2DKIE, @@ -2067,16 +2202,14 @@ from .tv2hu import ( from .tv4 import TV4IE from .tv5mondeplus import TV5MondePlusIE from .tv5unis import ( - TV5UnisVideoIE, TV5UnisIE, + TV5UnisVideoIE, ) -from .tva import ( - TVAIE, - QubIE, -) +from .tv24ua import TV24UAVideoIE +from .tva import TVAIE from .tvanouvelles import ( - TVANouvellesIE, TVANouvellesArticleIE, + TVANouvellesIE, ) from .tvc import ( TVCIE, @@ -2087,34 +2220,28 @@ from .tvigle import TvigleIE from .tviplayer import TVIPlayerIE from .tvland import TVLandIE from .tvn24 import TVN24IE -from .tvnet import TVNetIE from .tvnoe import TVNoeIE -from .tvnow import ( - TVNowIE, - TVNowFilmIE, - TVNowNewIE, - TVNowSeasonIE, - TVNowAnnualIE, - TVNowShowIE, -) from .tvopengr import ( - TVOpenGrWatchIE, TVOpenGrEmbedIE, + TVOpenGrWatchIE, ) from .tvp import ( - TVPEmbedIE, TVPIE, + TVPEmbedIE, TVPStreamIE, TVPVODSeriesIE, TVPVODVideoIE, ) from .tvplay import ( - TVPlayIE, TVPlayHomeIE, + TVPlayIE, ) from .tvplayer import TVPlayerIE +from .tvw import ( + TvwIE, + TvwTvChannelsIE, +) from .tweakers import TweakersIE -from .twentyfourvideo import TwentyFourVideoIE from .twentymin import TwentyMinutenIE from .twentythreevideo import TwentyThreeVideoIE from .twitcasting import ( @@ -2123,29 +2250,29 @@ from .twitcasting import ( TwitCastingUserIE, ) from .twitch import ( - TwitchVodIE, + TwitchClipsIE, TwitchCollectionIE, - TwitchVideosIE, + TwitchStreamIE, TwitchVideosClipsIE, TwitchVideosCollectionsIE, - TwitchStreamIE, - TwitchClipsIE, + TwitchVideosIE, + TwitchVodIE, ) from .twitter import ( - TwitterCardIE, - TwitterIE, TwitterAmplifyIE, TwitterBroadcastIE, - TwitterSpacesIE, + TwitterCardIE, + TwitterIE, TwitterShortenerIE, + TwitterSpacesIE, ) from .txxx import ( - TxxxIE, PornTopIE, + TxxxIE, ) from .udemy import ( + UdemyCourseIE, UdemyIE, - UdemyCourseIE ) from .udn import UDNEmbedIE from .ufctv import ( @@ -2154,17 +2281,17 @@ from .ufctv import ( ) from .ukcolumn import UkColumnIE from .uktvplay import UKTVPlayIE -from .digiteka import DigitekaIE -from .dlive import ( - DLiveVODIE, - DLiveStreamIE, +from .uliza import ( + UlizaPlayerIE, + UlizaPortalIE, ) -from .drooble import DroobleIE from .umg import UMGDeIE from .unistra import UnistraIE from .unity import UnityIE -from .unscripted import UnscriptedNewsVideoIE -from .unsupported import KnownDRMIE, KnownPiracyIE +from .unsupported import ( + KnownDRMIE, + KnownPiracyIE, +) from .uol import UOLIE from .uplynk import ( UplynkIE, @@ -2174,37 +2301,34 @@ from .urort import UrortIE from .urplay import URPlayIE from .usanetwork import USANetworkIE from .usatoday import USATodayIE -from .ustream import UstreamIE, UstreamChannelIE +from .ustream import ( + UstreamChannelIE, + UstreamIE, +) from .ustudio import ( - UstudioIE, UstudioEmbedIE, + UstudioIE, ) from .utreon import UtreonIE from .varzesh3 import Varzesh3IE from .vbox7 import Vbox7IE -from .veehd import VeeHDIE from .veo import VeoIE -from .veoh import ( - VeohIE, - VeohUserIE -) from .vesti import VestiIE from .vevo import ( VevoIE, VevoPlaylistIE, ) from .vgtv import ( + VGTVIE, BTArticleIE, BTVestlendingenIE, - VGTVIE, ) from .vh1 import VH1IE from .vice import ( - ViceIE, ViceArticleIE, + ViceIE, ViceShowIE, ) -from .vidbit import VidbitIE from .viddler import ViddlerIE from .videa import VideaIE from .videocampus_sachsen import ( @@ -2214,133 +2338,114 @@ from .videocampus_sachsen import ( from .videodetective import VideoDetectiveIE from .videofyme import VideofyMeIE from .videoken import ( + VideoKenCategoryIE, VideoKenIE, VideoKenPlayerIE, VideoKenPlaylistIE, - VideoKenCategoryIE, VideoKenTopicIE, ) from .videomore import ( VideomoreIE, - VideomoreVideoIE, VideomoreSeasonIE, + VideomoreVideoIE, ) from .videopress import VideoPressIE +from .vidflex import VidflexIE from .vidio import ( VidioIE, + VidioLiveIE, VidioPremierIE, - VidioLiveIE ) from .vidlii import VidLiiIE +from .vidly import VidlyIE +from .vidyard import VidyardIE from .viewlift import ( - ViewLiftIE, ViewLiftEmbedIE, + ViewLiftIE, ) from .viidea import ViideaIE from .vimeo import ( - VimeoIE, + VHXEmbedIE, VimeoAlbumIE, VimeoChannelIE, VimeoGroupsIE, + VimeoIE, VimeoLikesIE, VimeoOndemandIE, VimeoProIE, VimeoReviewIE, VimeoUserIE, VimeoWatchLaterIE, - VHXEmbedIE, ) from .vimm import ( VimmIE, VimmRecordingIE, ) -from .vimple import VimpleIE -from .vine import ( - VineIE, - VineUserIE, -) -from .viki import ( - VikiIE, - VikiChannelIE, -) +from .viously import ViouslyIE from .viqeo import ViqeoIE from .viu import ( ViuIE, - ViuPlaylistIE, ViuOTTIE, ViuOTTIndonesiaIE, + ViuPlaylistIE, ) from .vk import ( VKIE, - VKUserVideosIE, - VKWallPostIE, VKPlayIE, VKPlayLiveIE, + VKUserVideosIE, + VKWallPostIE, ) from .vocaroo import VocarooIE -from .vodlocker import VodlockerIE from .vodpl import VODPlIE from .vodplatform import VODPlatformIE -from .voicerepublic import VoiceRepublicIE from .voicy import ( - VoicyIE, VoicyChannelIE, + VoicyIE, ) from .volejtv import VolejTVIE -from .voot import ( - VootIE, - VootSeriesIE, -) from .voxmedia import ( - VoxMediaVolumeIE, VoxMediaIE, + VoxMediaVolumeIE, +) +from .vrsquare import ( + VrSquareChannelIE, + VrSquareIE, + VrSquareSearchIE, + VrSquareSectionIE, ) from .vrt import ( VRTIE, - VrtNUIE, - KetnetIE, DagelijkseKostIE, + Radio1BeIE, + VrtNUIE, ) -from .vrak import VrakIE -from .vrv import ( - VRVIE, - VRVSeriesIE, -) -from .vshare import VShareIE from .vtm import VTMIE -from .medialaan import MedialaanIE +from .vtv import ( + VTVIE, + VTVGoIE, +) from .vuclip import VuClipIE -from .vupload import VuploadIE from .vvvvid import ( VVVVIDIE, VVVVIDShowIE, ) -from .vyborymos import VyboryMosIE -from .vzaar import VzaarIE -from .wakanim import WakanimIE from .walla import WallaIE from .washingtonpost import ( - WashingtonPostIE, WashingtonPostArticleIE, -) -from .wasdtv import ( - WASDTVStreamIE, - WASDTVRecordIE, - WASDTVClipIE, + WashingtonPostIE, ) from .wat import WatIE -from .watchbox import WatchBoxIE -from .watchindianporn import WatchIndianPornIE from .wdr import ( WDRIE, - WDRPageIE, WDRElefantIE, WDRMobileIE, + WDRPageIE, ) from .webcamerapl import WebcameraplIE from .webcaster import ( - WebcasterIE, WebcasterFeedIE, + WebcasterIE, ) from .webofstories import ( WebOfStoriesIE, @@ -2348,42 +2453,42 @@ from .webofstories import ( ) from .weibo import ( WeiboIE, - WeiboMobileIE + WeiboUserIE, + WeiboVideoIE, ) from .weiqitv import WeiqiTVIE from .weverse import ( WeverseIE, - WeverseMediaIE, - WeverseMomentIE, - WeverseLiveTabIE, - WeverseMediaTabIE, WeverseLiveIE, + WeverseLiveTabIE, + WeverseMediaIE, + WeverseMediaTabIE, + WeverseMomentIE, ) from .wevidi import WeVidiIE from .weyyak import WeyyakIE +from .whowatch import WhoWatchIE from .whyp import WhypIE from .wikimedia import WikimediaIE -from .willow import WillowIE from .wimbledon import WimbledonIE from .wimtv import WimTVIE -from .whowatch import WhoWatchIE from .wistia import ( + WistiaChannelIE, WistiaIE, WistiaPlaylistIE, - WistiaChannelIE, ) from .wordpress import ( - WordpressPlaylistEmbedIE, WordpressMiniAudioPlayerEmbedIE, + WordpressPlaylistEmbedIE, ) from .worldstarhiphop import WorldStarHipHopIE from .wppilot import ( - WPPilotIE, WPPilotChannelsIE, + WPPilotIE, ) from .wrestleuniverse import ( - WrestleUniverseVODIE, WrestleUniversePPVIE, + WrestleUniverseVODIE, ) from .wsj import ( WSJIE, @@ -2391,89 +2496,91 @@ from .wsj import ( ) from .wwe import WWEIE from .wykop import ( - WykopDigIE, WykopDigCommentIE, - WykopPostIE, + WykopDigIE, WykopPostCommentIE, + WykopPostIE, ) from .xanimu import XanimuIE -from .xbef import XBefIE from .xboxclips import XboxClipsIE -from .xfileshare import XFileShareIE from .xhamster import ( - XHamsterIE, XHamsterEmbedIE, + XHamsterIE, XHamsterUserIE, ) +from .xiaohongshu import XiaoHongShuIE from .ximalaya import ( + XimalayaAlbumIE, XimalayaIE, - XimalayaAlbumIE ) from .xinpianchang import XinpianchangIE from .xminus import XMinusIE from .xnxx import XNXXIE from .xstream import XstreamIE -from .xtube import XTubeUserIE, XTubeIE -from .xuite import XuiteIE from .xvideos import ( XVideosIE, - XVideosQuickiesIE + XVideosQuickiesIE, ) from .xxxymovies import XXXYMoviesIE from .yahoo import ( YahooIE, - YahooSearchIE, YahooJapanNewsIE, + YahooSearchIE, ) from .yandexdisk import YandexDiskIE from .yandexmusic import ( - YandexMusicTrackIE, YandexMusicAlbumIE, - YandexMusicPlaylistIE, - YandexMusicArtistTracksIE, YandexMusicArtistAlbumsIE, + YandexMusicArtistTracksIE, + YandexMusicPlaylistIE, + YandexMusicTrackIE, ) from .yandexvideo import ( YandexVideoIE, YandexVideoPreviewIE, - ZenYandexIE, ZenYandexChannelIE, + ZenYandexIE, ) from .yapfiles import YapFilesIE from .yappy import ( YappyIE, YappyProfileIE, ) -from .yesjapan import YesJapanIE -from .yinyuetai import YinYueTaiIE from .yle_areena import YleAreenaIE -from .ynet import YnetIE from .youjizz import YouJizzIE from .youku import ( YoukuIE, YoukuShowIE, ) from .younow import ( - YouNowLiveIE, YouNowChannelIE, + YouNowLiveIE, YouNowMomentIE, ) -from .youporn import YouPornIE -from .yourporn import YourPornIE -from .yourupload import YourUploadIE +from .youporn import ( + YouPornCategoryIE, + YouPornChannelIE, + YouPornCollectionIE, + YouPornIE, + YouPornStarIE, + YouPornTagIE, + YouPornVideosIE, +) from .zaiko import ( - ZaikoIE, ZaikoETicketIE, + ZaikoIE, ) from .zapiks import ZapiksIE from .zattoo import ( BBVTVIE, + EWETVIE, + SAKTVIE, + VTXTVIE, BBVTVLiveIE, BBVTVRecordingsIE, EinsUndEinsTVIE, EinsUndEinsTVLiveIE, EinsUndEinsTVRecordingsIE, - EWETVIE, EWETVLiveIE, EWETVRecordingsIE, GlattvisionTVIE, @@ -2491,13 +2598,11 @@ from .zattoo import ( QuantumTVIE, QuantumTVLiveIE, QuantumTVRecordingsIE, + SAKTVLiveIE, + SAKTVRecordingsIE, SaltTVIE, SaltTVLiveIE, SaltTVRecordingsIE, - SAKTVIE, - SAKTVLiveIE, - SAKTVRecordingsIE, - VTXTVIE, VTXTVLiveIE, VTXTVRecordingsIE, WalyTVIE, @@ -2508,21 +2613,29 @@ from .zattoo import ( ZattooMoviesIE, ZattooRecordingsIE, ) -from .zdf import ZDFIE, ZDFChannelIE +from .zdf import ( + ZDFIE, + ZDFChannelIE, +) from .zee5 import ( Zee5IE, Zee5SeriesIE, ) from .zeenews import ZeeNewsIE +from .zenporn import ZenPornIE +from .zetland import ZetlandDKArticleIE from .zhihu import ZhihuIE from .zingmp3 import ( - ZingMp3IE, ZingMp3AlbumIE, ZingMp3ChartHomeIE, - ZingMp3WeekChartIE, ZingMp3ChartMusicVideoIE, - ZingMp3UserIE, ZingMp3HubIE, + ZingMp3IE, + ZingMp3LiveRadioIE, + ZingMp3PodcastEpisodeIE, + ZingMp3PodcastIE, + ZingMp3UserIE, + ZingMp3WeekChartIE, ) from .zoom import ZoomIE from .zype import ZypeIE diff --git a/plugins/youtube_download/yt_dlp/extractor/abc.py b/plugins/youtube_download/yt_dlp/extractor/abc.py index f56133eb..7296be73 100644 --- a/plugins/youtube_download/yt_dlp/extractor/abc.py +++ b/plugins/youtube_download/yt_dlp/extractor/abc.py @@ -4,18 +4,18 @@ import re import time from .common import InfoExtractor -from ..compat import compat_str from ..utils import ( - dict_get, ExtractorError, - js_to_json, + dict_get, int_or_none, + js_to_json, parse_iso8601, str_or_none, traverse_obj, try_get, unescapeHTML, update_url_query, + url_or_none, ) @@ -66,7 +66,7 @@ class ABCIE(InfoExtractor): 'ext': 'mp4', 'title': 'WWI Centenary', 'description': 'md5:c2379ec0ca84072e86b446e536954546', - } + }, }, { 'url': 'https://www.abc.net.au/news/programs/the-world/2020-06-10/black-lives-matter-protests-spawn-support-for/12342074', 'info_dict': { @@ -74,7 +74,7 @@ class ABCIE(InfoExtractor): 'ext': 'mp4', 'title': 'Black Lives Matter protests spawn support for Papuans in Indonesia', 'description': 'md5:2961a17dc53abc558589ccd0fb8edd6f', - } + }, }, { 'url': 'https://www.abc.net.au/btn/newsbreak/btn-newsbreak-20200814/12560476', 'info_dict': { @@ -85,7 +85,7 @@ class ABCIE(InfoExtractor): 'upload_date': '20200813', 'uploader': 'Behind the News', 'uploader_id': 'behindthenews', - } + }, }, { 'url': 'https://www.abc.net.au/news/2023-06-25/wagner-boss-orders-troops-back-to-bases-to-avoid-bloodshed/102520540', 'info_dict': { @@ -94,7 +94,7 @@ class ABCIE(InfoExtractor): 'ext': 'mp4', 'description': 'Wagner troops leave Rostov-on-Don and\xa0Yevgeny Prigozhin will move to Belarus under a deal brokered by Belarusian President Alexander Lukashenko to end the mutiny.', 'thumbnail': 'https://live-production.wcms.abc-cdn.net.au/0c170f5b57f0105c432f366c0e8e267b?impolicy=wcms_crop_resize&cropH=2813&cropW=5000&xPos=0&yPos=249&width=862&height=485', - } + }, }] def _real_extract(self, url): @@ -125,7 +125,7 @@ class ABCIE(InfoExtractor): if mobj is None: expired = self._html_search_regex(r'(?s)class="expired-(?:video|audio)".+?(.+?)', webpage, 'expired', None) if expired: - raise ExtractorError('%s said: %s' % (self.IE_NAME, expired), expected=True) + raise ExtractorError(f'{self.IE_NAME} said: {expired}', expected=True) raise ExtractorError('Unable to extract video urls') urls_info = self._parse_json( @@ -163,7 +163,7 @@ class ABCIE(InfoExtractor): 'height': height, 'tbr': bitrate, 'filesize': int_or_none(url_info.get('filesize')), - 'format_id': format_id + 'format_id': format_id, }) return { @@ -180,20 +180,100 @@ class ABCIViewIE(InfoExtractor): _VALID_URL = r'https?://iview\.abc\.net\.au/(?:[^/]+/)*video/(?P[^/?#]+)' _GEO_COUNTRIES = ['AU'] - # ABC iview programs are normally available for 14 days only. _TESTS = [{ + 'url': 'https://iview.abc.net.au/show/utopia/series/1/video/CO1211V001S00', + 'md5': '52a942bfd7a0b79a6bfe9b4ce6c9d0ed', + 'info_dict': { + 'id': 'CO1211V001S00', + 'ext': 'mp4', + 'title': 'Series 1 Ep 1 Wood For The Trees', + 'series': 'Utopia', + 'description': 'md5:0cfb2c183c1b952d1548fd65c8a95c00', + 'upload_date': '20230726', + 'uploader_id': 'abc1', + 'series_id': 'CO1211V', + 'episode_id': 'CO1211V001S00', + 'season_number': 1, + 'season': 'Season 1', + 'episode_number': 1, + 'episode': 'Wood For The Trees', + 'thumbnail': 'https://cdn.iview.abc.net.au/thumbs/i/co/CO1211V001S00_5ad8353f4df09_1280.jpg', + 'timestamp': 1690403700, + }, + 'params': { + 'skip_download': True, + }, + }, { + 'note': 'No episode name', 'url': 'https://iview.abc.net.au/show/gruen/series/11/video/LE1927H001S00', 'md5': '67715ce3c78426b11ba167d875ac6abf', 'info_dict': { 'id': 'LE1927H001S00', 'ext': 'mp4', - 'title': "Series 11 Ep 1", - 'series': "Gruen", + 'title': 'Series 11 Ep 1', + 'series': 'Gruen', 'description': 'md5:52cc744ad35045baf6aded2ce7287f67', 'upload_date': '20190925', 'uploader_id': 'abc1', + 'series_id': 'LE1927H', + 'episode_id': 'LE1927H001S00', + 'season_number': 11, + 'season': 'Season 11', + 'episode_number': 1, + 'episode': 'Episode 1', + 'thumbnail': 'https://cdn.iview.abc.net.au/thumbs/i/le/LE1927H001S00_5d954fbd79e25_1280.jpg', 'timestamp': 1569445289, }, + 'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'], + 'params': { + 'skip_download': True, + }, + }, { + 'note': 'No episode number', + 'url': 'https://iview.abc.net.au/show/four-corners/series/2022/video/NC2203H039S00', + 'md5': '77cb7d8434440e3b28fbebe331c2456a', + 'info_dict': { + 'id': 'NC2203H039S00', + 'ext': 'mp4', + 'title': 'Series 2022 Locking Up Kids', + 'series': 'Four Corners', + 'description': 'md5:54829ca108846d1a70e1fcce2853e720', + 'upload_date': '20221114', + 'uploader_id': 'abc1', + 'series_id': 'NC2203H', + 'episode_id': 'NC2203H039S00', + 'season_number': 2022, + 'season': 'Season 2022', + 'episode': 'Locking Up Kids', + 'thumbnail': 'https://cdn.iview.abc.net.au/thumbs/i/nc/NC2203H039S00_636d8a0944a22_1920.jpg', + 'timestamp': 1668460497, + + }, + 'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'], + 'params': { + 'skip_download': True, + }, + }, { + 'note': 'No episode name or number', + 'url': 'https://iview.abc.net.au/show/landline/series/2021/video/RF2004Q043S00', + 'md5': '2e17dec06b13cc81dc119d2565289396', + 'info_dict': { + 'id': 'RF2004Q043S00', + 'ext': 'mp4', + 'title': 'Series 2021', + 'series': 'Landline', + 'description': 'md5:c9f30d9c0c914a7fd23842f6240be014', + 'upload_date': '20211205', + 'uploader_id': 'abc1', + 'series_id': 'RF2004Q', + 'episode_id': 'RF2004Q043S00', + 'season_number': 2021, + 'season': 'Season 2021', + 'thumbnail': 'https://cdn.iview.abc.net.au/thumbs/i/rf/RF2004Q043S00_61a950639dbc0_1920.jpg', + 'timestamp': 1638710705, + + }, + 'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'], 'params': { 'skip_download': True, }, @@ -207,13 +287,12 @@ class ABCIViewIE(InfoExtractor): stream = next(s for s in video_params['playlist'] if s.get('type') in ('program', 'livestream')) house_number = video_params.get('episodeHouseNumber') or video_id - path = '/auth/hls/sign?ts={0}&hn={1}&d=android-tablet'.format( - int(time.time()), house_number) + path = f'/auth/hls/sign?ts={int(time.time())}&hn={house_number}&d=android-tablet' sig = hmac.new( b'android.content.res.Resources', - path.encode('utf-8'), hashlib.sha256).hexdigest() + path.encode(), hashlib.sha256).hexdigest() token = self._download_webpage( - 'http://iview.abc.net.au{0}&sig={1}'.format(path, sig), video_id) + f'http://iview.abc.net.au{path}&sig={sig}', video_id) def tokenize_url(url, token): return update_url_query(url, { @@ -222,7 +301,7 @@ class ABCIViewIE(InfoExtractor): for sd in ('1080', '720', 'sd', 'sd-low'): sd_url = try_get( - stream, lambda x: x['streams']['hls'][sd], compat_str) + stream, lambda x: x['streams']['hls'][sd], str) if not sd_url: continue formats = self._extract_m3u8_formats( @@ -255,6 +334,8 @@ class ABCIViewIE(InfoExtractor): 'episode_number': int_or_none(self._search_regex( r'\bEp\s+(\d+)\b', title, 'episode number', default=None)), 'episode_id': house_number, + 'episode': self._search_regex( + r'^(?:Series\s+\d+)?\s*(?:Ep\s+\d+)?\s*(.*)$', title, 'episode', default='') or None, 'uploader_id': video_params.get('channel'), 'formats': formats, 'subtitles': subtitles, @@ -275,7 +356,7 @@ class ABCIViewShowSeriesIE(InfoExtractor): 'description': 'md5:93119346c24a7c322d446d8eece430ff', 'series': 'Upper Middle Bogan', 'season': 'Series 1', - 'thumbnail': r're:^https?://cdn\.iview\.abc\.net\.au/thumbs/.*\.jpg$' + 'thumbnail': r're:^https?://cdn\.iview\.abc\.net\.au/thumbs/.*\.jpg$', }, 'playlist_count': 8, }, { @@ -294,17 +375,39 @@ class ABCIViewShowSeriesIE(InfoExtractor): 'noplaylist': True, 'skip_download': 'm3u8', }, + }, { + # 'videoEpisodes' is a dict with `items` key + 'url': 'https://iview.abc.net.au/show/7-30-mark-humphries-satire', + 'info_dict': { + 'id': '178458-0', + 'title': 'Episodes', + 'description': 'Satirist Mark Humphries brings his unique perspective on current political events for 7.30.', + 'series': '7.30 Mark Humphries Satire', + 'season': 'Episodes', + 'thumbnail': r're:^https?://cdn\.iview\.abc\.net\.au/thumbs/.*\.jpg$', + }, + 'playlist_count': 15, + 'skip': 'This program is not currently available in ABC iview', + }, { + 'url': 'https://iview.abc.net.au/show/inbestigators', + 'info_dict': { + 'id': '175343-1', + 'title': 'Series 1', + 'description': 'md5:b9976935a6450e5b78ce2a940a755685', + 'series': 'The Inbestigators', + 'season': 'Series 1', + 'thumbnail': r're:^https?://cdn\.iview\.abc\.net\.au/thumbs/.+\.jpg', + }, + 'playlist_count': 17, }] def _real_extract(self, url): show_id = self._match_id(url) webpage = self._download_webpage(url, show_id) - webpage_data = self._search_regex( - r'window\.__INITIAL_STATE__\s*=\s*[\'"](.+?)[\'"]\s*;', - webpage, 'initial state') - video_data = self._parse_json( - unescapeHTML(webpage_data).encode('utf-8').decode('unicode_escape'), show_id) - video_data = video_data['route']['pageData']['_embedded'] + video_data = self._search_json( + r'window\.__INITIAL_STATE__\s*=\s*[\'"]', webpage, 'initial state', show_id, + transform_source=lambda x: x.encode().decode('unicode_escape'), + end_pattern=r'[\'"]\s*;')['route']['pageData']['_embedded'] highlight = try_get(video_data, lambda x: x['highlightVideo']['shareUrl']) if not self._yes_playlist(show_id, bool(highlight), video_label='highlight video'): @@ -313,12 +416,14 @@ class ABCIViewShowSeriesIE(InfoExtractor): series = video_data['selectedSeries'] return { '_type': 'playlist', - 'entries': [self.url_result(episode['shareUrl']) - for episode in series['_embedded']['videoEpisodes']], + 'entries': [self.url_result(episode_url, ABCIViewIE) + for episode_url in traverse_obj(series, ( + '_embedded', 'videoEpisodes', (None, 'items'), ..., 'shareUrl', {url_or_none}))], 'id': series.get('id'), 'title': dict_get(series, ('title', 'displaySubtitle')), 'description': series.get('description'), 'series': dict_get(series, ('showTitle', 'displayTitle')), 'season': dict_get(series, ('title', 'displaySubtitle')), - 'thumbnail': series.get('thumbnail'), + 'thumbnail': traverse_obj( + series, 'thumbnail', ('images', lambda _, v: v['name'] == 'seriesThumbnail', 'url'), get_all=False), } diff --git a/plugins/youtube_download/yt_dlp/extractor/abcnews.py b/plugins/youtube_download/yt_dlp/extractor/abcnews.py index a57295b1..7215500b 100644 --- a/plugins/youtube_download/yt_dlp/extractor/abcnews.py +++ b/plugins/youtube_download/yt_dlp/extractor/abcnews.py @@ -58,7 +58,7 @@ class AbcNewsVideoIE(AMPIE): display_id = mobj.group('display_id') video_id = mobj.group('id') info_dict = self._extract_feed_info( - 'http://abcnews.go.com/video/itemfeed?id=%s' % video_id) + f'http://abcnews.go.com/video/itemfeed?id={video_id}') info_dict.update({ 'id': video_id, 'display_id': display_id, diff --git a/plugins/youtube_download/yt_dlp/extractor/abcotvs.py b/plugins/youtube_download/yt_dlp/extractor/abcotvs.py index 6dca19de..ea5882b2 100644 --- a/plugins/youtube_download/yt_dlp/extractor/abcotvs.py +++ b/plugins/youtube_download/yt_dlp/extractor/abcotvs.py @@ -1,5 +1,4 @@ from .common import InfoExtractor -from ..compat import compat_str from ..utils import ( dict_get, int_or_none, @@ -57,11 +56,11 @@ class ABCOTVSIE(InfoExtractor): data = self._download_json( 'https://api.abcotvs.com/v2/content', display_id, query={ 'id': video_id, - 'key': 'otv.web.%s.story' % station, + 'key': f'otv.web.{station}.story', 'station': station, })['data'] video = try_get(data, lambda x: x['featuredMedia']['video'], dict) or data - video_id = compat_str(dict_get(video, ('id', 'publishedKey'), video_id)) + video_id = str(dict_get(video, ('id', 'publishedKey'), video_id)) title = video.get('title') or video['linkText'] formats = [] diff --git a/plugins/youtube_download/yt_dlp/extractor/abematv.py b/plugins/youtube_download/yt_dlp/extractor/abematv.py index 163b83c6..8f2fc4c8 100644 --- a/plugins/youtube_download/yt_dlp/extractor/abematv.py +++ b/plugins/youtube_download/yt_dlp/extractor/abematv.py @@ -6,53 +6,54 @@ import hmac import io import json import re -import struct import time import urllib.parse -import urllib.request -import urllib.response import uuid from .common import InfoExtractor from ..aes import aes_ecb_decrypt +from ..networking import RequestHandler, Response +from ..networking.exceptions import TransportError from ..utils import ( ExtractorError, - bytes_to_intlist, + OnDemandPagedList, decode_base_n, int_or_none, - intlist_to_bytes, - OnDemandPagedList, time_seconds, traverse_obj, + update_url, update_url_query, ) -def add_opener(ydl, handler): # FIXME: Create proper API in .networking - """Add a handler for opening URLs, like _download_webpage""" - # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L426 - # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L605 - rh = ydl._request_director.handlers['Urllib'] - if 'abematv-license' in rh._SUPPORTED_URL_SCHEMES: - return - opener = rh._get_instance(cookiejar=ydl.cookiejar, proxies=ydl.proxies) - assert isinstance(opener, urllib.request.OpenerDirector) - opener.add_handler(handler) - rh._SUPPORTED_URL_SCHEMES = (*rh._SUPPORTED_URL_SCHEMES, 'abematv-license') +class AbemaLicenseRH(RequestHandler): + _SUPPORTED_URL_SCHEMES = ('abematv-license',) + _SUPPORTED_PROXY_SCHEMES = None + _SUPPORTED_FEATURES = None + RH_NAME = 'abematv_license' + _STRTABLE = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' + _HKEY = b'3AF0298C219469522A313570E8583005A642E73EDD58E3EA2FB7339D3DF1597E' -class AbemaLicenseHandler(urllib.request.BaseHandler): - handler_order = 499 - STRTABLE = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' - HKEY = b'3AF0298C219469522A313570E8583005A642E73EDD58E3EA2FB7339D3DF1597E' - - def __init__(self, ie: 'AbemaTVIE'): - # the protocol that this should really handle is 'abematv-license://' - # abematv_license_open is just a placeholder for development purposes - # ref. https://github.com/python/cpython/blob/f4c03484da59049eb62a9bf7777b963e2267d187/Lib/urllib/request.py#L510 - setattr(self, 'abematv-license_open', getattr(self, 'abematv_license_open')) + def __init__(self, *, ie: 'AbemaTVIE', **kwargs): + super().__init__(**kwargs) self.ie = ie + def _send(self, request): + url = request.url + ticket = urllib.parse.urlparse(url).netloc + + try: + response_data = self._get_videokey_from_ticket(ticket) + except ExtractorError as e: + raise TransportError(cause=e.cause) from e + except (IndexError, KeyError, TypeError) as e: + raise TransportError(cause=repr(e)) from e + + return Response( + io.BytesIO(response_data), url, + headers={'Content-Length': str(len(response_data))}) + def _get_videokey_from_ticket(self, ticket): to_show = self.ie.get_param('verbose', False) media_token = self.ie._get_media_token(to_show=to_show) @@ -62,33 +63,27 @@ class AbemaLicenseHandler(urllib.request.BaseHandler): query={'t': media_token}, data=json.dumps({ 'kv': 'a', - 'lt': ticket - }).encode('utf-8'), + 'lt': ticket, + }).encode(), headers={ 'Content-Type': 'application/json', }) - res = decode_base_n(license_response['k'], table=self.STRTABLE) - encvideokey = bytes_to_intlist(struct.pack('>QQ', res >> 64, res & 0xffffffffffffffff)) + res = decode_base_n(license_response['k'], table=self._STRTABLE) + encvideokey = list(res.to_bytes(16, 'big')) h = hmac.new( - binascii.unhexlify(self.HKEY), - (license_response['cid'] + self.ie._DEVICE_ID).encode('utf-8'), + binascii.unhexlify(self._HKEY), + (license_response['cid'] + self.ie._DEVICE_ID).encode(), digestmod=hashlib.sha256) - enckey = bytes_to_intlist(h.digest()) + enckey = list(h.digest()) - return intlist_to_bytes(aes_ecb_decrypt(encvideokey, enckey)) - - def abematv_license_open(self, url): - url = url.get_full_url() if isinstance(url, urllib.request.Request) else url - ticket = urllib.parse.urlparse(url).netloc - response_data = self._get_videokey_from_ticket(ticket) - return urllib.response.addinfourl(io.BytesIO(response_data), headers={ - 'Content-Length': str(len(response_data)), - }, url=url, code=200) + return bytes(aes_ecb_decrypt(encvideokey, enckey)) class AbemaTVBaseIE(InfoExtractor): + _NETRC_MACHINE = 'abematv' + _USERTOKEN = None _DEVICE_ID = None _MEDIATOKEN = None @@ -97,11 +92,11 @@ class AbemaTVBaseIE(InfoExtractor): @classmethod def _generate_aks(cls, deviceid): - deviceid = deviceid.encode('utf-8') + deviceid = deviceid.encode() # add 1 hour and then drop minute and secs ts_1hour = int((time_seconds() // 3600 + 1) * 3600) time_struct = time.gmtime(ts_1hour) - ts_1hour_str = str(ts_1hour).encode('utf-8') + ts_1hour_str = str(ts_1hour).encode() tmp = None @@ -113,7 +108,7 @@ class AbemaTVBaseIE(InfoExtractor): def mix_tmp(count): nonlocal tmp - for i in range(count): + for _ in range(count): mix_once(tmp) def mix_twist(nonce): @@ -133,11 +128,15 @@ class AbemaTVBaseIE(InfoExtractor): if self._USERTOKEN: return self._USERTOKEN + self._downloader._request_director.add_handler(AbemaLicenseRH(ie=self, logger=None)) + username, _ = self._get_login_info() - AbemaTVBaseIE._USERTOKEN = username and self.cache.load(self._NETRC_MACHINE, username) + auth_cache = username and self.cache.load(self._NETRC_MACHINE, username, min_ver='2024.01.19') + AbemaTVBaseIE._USERTOKEN = auth_cache and auth_cache.get('usertoken') if AbemaTVBaseIE._USERTOKEN: # try authentication with locally stored token try: + AbemaTVBaseIE._DEVICE_ID = auth_cache.get('device_id') self._get_media_token(True) return except ExtractorError as e: @@ -150,13 +149,12 @@ class AbemaTVBaseIE(InfoExtractor): data=json.dumps({ 'deviceId': self._DEVICE_ID, 'applicationKeySecret': aks, - }).encode('utf-8'), + }).encode(), headers={ 'Content-Type': 'application/json', }) AbemaTVBaseIE._USERTOKEN = user_data['token'] - add_opener(self._downloader, AbemaLicenseHandler(self)) return self._USERTOKEN def _get_media_token(self, invalidate=False, to_show=True): @@ -171,13 +169,44 @@ class AbemaTVBaseIE(InfoExtractor): 'osLang': 'ja_JP', 'osTimezone': 'Asia/Tokyo', 'appId': 'tv.abema', - 'appVersion': '3.27.1' + 'appVersion': '3.27.1', }, headers={ 'Authorization': f'bearer {self._get_device_token()}', })['token'] return self._MEDIATOKEN + def _perform_login(self, username, password): + self._get_device_token() + if self.cache.load(self._NETRC_MACHINE, username, min_ver='2024.01.19') and self._get_media_token(): + self.write_debug('Skipping logging in') + return + + if '@' in username: # don't strictly check if it's email address or not + ep, method = 'user/email', 'email' + else: + ep, method = 'oneTimePassword', 'userId' + + login_response = self._download_json( + f'https://api.abema.io/v1/auth/{ep}', None, note='Logging in', + data=json.dumps({ + method: username, + 'password': password, + }).encode(), headers={ + 'Authorization': f'bearer {self._get_device_token()}', + 'Origin': 'https://abema.tv', + 'Referer': 'https://abema.tv/', + 'Content-Type': 'application/json', + }) + + AbemaTVBaseIE._USERTOKEN = login_response['token'] + self._get_media_token(True) + auth_cache = { + 'device_id': AbemaTVBaseIE._DEVICE_ID, + 'usertoken': AbemaTVBaseIE._USERTOKEN, + } + self.cache.store(self._NETRC_MACHINE, username, auth_cache) + def _call_api(self, endpoint, video_id, query=None, note='Downloading JSON metadata'): return self._download_json( f'https://api.abema.io/{endpoint}', video_id, query=query or {}, @@ -201,14 +230,14 @@ class AbemaTVBaseIE(InfoExtractor): class AbemaTVIE(AbemaTVBaseIE): _VALID_URL = r'https?://abema\.tv/(?Pnow-on-air|video/episode|channels/.+?/slots)/(?P[^?/]+)' - _NETRC_MACHINE = 'abematv' _TESTS = [{ 'url': 'https://abema.tv/video/episode/194-25_s2_p1', 'info_dict': { 'id': '194-25_s2_p1', 'title': '第1話 「チーズケーキ」　「モーニング再び」', 'series': '異世界食堂２', - 'series_number': 2, + 'season': 'シーズン2', + 'season_number': 2, 'episode': '第1話 「チーズケーキ」　「モーニング再び」', 'episode_number': 1, }, @@ -220,7 +249,7 @@ class AbemaTVIE(AbemaTVBaseIE): 'title': 'ゆるキャン△ SEASON２ 全話一挙【無料ビデオ72時間】', 'series': 'ゆるキャン△ SEASON２', 'episode': 'ゆるキャン△ SEASON２ 全話一挙【無料ビデオ72時間】', - 'series_number': 2, + 'season_number': 2, 'episode_number': 1, 'description': 'md5:9c5a3172ae763278f9303922f0ea5b17', }, @@ -249,33 +278,6 @@ class AbemaTVIE(AbemaTVBaseIE): }] _TIMETABLE = None - def _perform_login(self, username, password): - self._get_device_token() - if self.cache.load(self._NETRC_MACHINE, username) and self._get_media_token(): - self.write_debug('Skipping logging in') - return - - if '@' in username: # don't strictly check if it's email address or not - ep, method = 'user/email', 'email' - else: - ep, method = 'oneTimePassword', 'userId' - - login_response = self._download_json( - f'https://api.abema.io/v1/auth/{ep}', None, note='Logging in', - data=json.dumps({ - method: username, - 'password': password - }).encode('utf-8'), headers={ - 'Authorization': f'bearer {self._get_device_token()}', - 'Origin': 'https://abema.tv', - 'Referer': 'https://abema.tv/', - 'Content-Type': 'application/json', - }) - - AbemaTVBaseIE._USERTOKEN = login_response['token'] - self._get_media_token(True) - self.cache.store(self._NETRC_MACHINE, username, AbemaTVBaseIE._USERTOKEN) - def _real_extract(self, url): # starting download using infojson from this extractor is undefined behavior, # and never be fixed in the future; you must trigger downloads by directly specifying URL. @@ -331,7 +333,7 @@ class AbemaTVIE(AbemaTVBaseIE): description = self._html_search_regex( (r'(.+?)

(.+?)(.+?)[^?/]+)' + _VALID_URL = r'https?://abema\.tv/video/title/(?P[^?/#]+)/?(?:\?(?:[^#]+&)?s=(?P[^&#]+))?' _PAGE_SIZE = 25 _TESTS = [{ - 'url': 'https://abema.tv/video/title/90-1597', + 'url': 'https://abema.tv/video/title/90-1887', 'info_dict': { - 'id': '90-1597', + 'id': '90-1887', 'title': 'シャッフルアイランド', + 'description': 'md5:61b2425308f41a5282a926edda66f178', }, 'playlist_mincount': 2, }, { @@ -432,41 +442,54 @@ class AbemaTVTitleIE(AbemaTVBaseIE): 'info_dict': { 'id': '193-132', 'title': '真心が届く~僕とスターのオフィス・ラブ!?~', + 'description': 'md5:9b59493d1f3a792bafbc7319258e7af8', }, 'playlist_mincount': 16, }, { - 'url': 'https://abema.tv/video/title/25-102', + 'url': 'https://abema.tv/video/title/25-1nzan-whrxe', 'info_dict': { - 'id': '25-102', - 'title': 'ソードアート・オンライン アリシゼーション', + 'id': '25-1nzan-whrxe', + 'title': 'ソードアート・オンライン', + 'description': 'md5:c094904052322e6978495532bdbf06e6', }, - 'playlist_mincount': 24, + 'playlist_mincount': 25, + }, { + 'url': 'https://abema.tv/video/title/26-2mzbynr-cph?s=26-2mzbynr-cph_s40', + 'info_dict': { + 'title': '〈物語〉シリーズ', + 'id': '26-2mzbynr-cph', + 'description': 'md5:e67873de1c88f360af1f0a4b84847a52', + }, + 'playlist_count': 59, }] - def _fetch_page(self, playlist_id, series_version, page): + def _fetch_page(self, playlist_id, series_version, season_id, page): + query = { + 'seriesVersion': series_version, + 'offset': str(page * self._PAGE_SIZE), + 'order': 'seq', + 'limit': str(self._PAGE_SIZE), + } + if season_id: + query['seasonId'] = season_id programs = self._call_api( f'v1/video/series/{playlist_id}/programs', playlist_id, note=f'Downloading page {page + 1}', - query={ - 'seriesVersion': series_version, - 'offset': str(page * self._PAGE_SIZE), - 'order': 'seq', - 'limit': str(self._PAGE_SIZE), - }) + query=query) yield from ( self.url_result(f'https://abema.tv/video/episode/{x}') for x in traverse_obj(programs, ('programs', ..., 'id'))) - def _entries(self, playlist_id, series_version): + def _entries(self, playlist_id, series_version, season_id): return OnDemandPagedList( - functools.partial(self._fetch_page, playlist_id, series_version), + functools.partial(self._fetch_page, playlist_id, series_version, season_id), self._PAGE_SIZE) def _real_extract(self, url): - playlist_id = self._match_id(url) + playlist_id, season_id = self._match_valid_url(url).group('id', 'season') series_info = self._call_api(f'v1/video/series/{playlist_id}', playlist_id) return self.playlist_result( - self._entries(playlist_id, series_info['version']), playlist_id=playlist_id, + self._entries(playlist_id, series_info['version'], season_id), playlist_id=playlist_id, playlist_title=series_info.get('title'), playlist_description=series_info.get('content')) diff --git a/plugins/youtube_download/yt_dlp/extractor/academicearth.py b/plugins/youtube_download/yt_dlp/extractor/academicearth.py index d9691cb5..b997a028 100644 --- a/plugins/youtube_download/yt_dlp/extractor/academicearth.py +++ b/plugins/youtube_download/yt_dlp/extractor/academicearth.py @@ -4,7 +4,7 @@ from .common import InfoExtractor class AcademicEarthCourseIE(InfoExtractor): - _VALID_URL = r'^https?://(?:www\.)?academicearth\.org/playlists/(?P[^?#/]+)' + _VALID_URL = r'https?://(?:www\.)?academicearth\.org/playlists/(?P[^?#/]+)' IE_NAME = 'AcademicEarth:Course' _TEST = { 'url': 'http://academicearth.org/playlists/laws-of-nature/', diff --git a/plugins/youtube_download/yt_dlp/extractor/acast.py b/plugins/youtube_download/yt_dlp/extractor/acast.py index 427d04c3..eb467cb7 100644 --- a/plugins/youtube_download/yt_dlp/extractor/acast.py +++ b/plugins/youtube_download/yt_dlp/extractor/acast.py @@ -43,14 +43,14 @@ class ACastIE(ACastBaseIE): _VALID_URL = r'''(?x: https?:// (?: - (?:(?:embed|www)\.)?acast\.com/| + (?:(?:embed|www|shows)\.)?acast\.com/| play\.acast\.com/s/ ) - (?P[^/]+)/(?P[^/#?"]+) + (?P[^/?#]+)/(?:episodes/)?(?P[^/#?"]+) )''' _EMBED_REGEX = [rf'(?x)]+\bsrc=[\'"](?P{_VALID_URL})'] _TESTS = [{ - 'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna', + 'url': 'https://shows.acast.com/sparpodcast/episodes/2.raggarmordet-rosterurdetforflutna', 'info_dict': { 'id': '2a92b283-1a75-4ad8-8396-499c641de0d9', 'ext': 'mp3', @@ -59,7 +59,7 @@ class ACastIE(ACastBaseIE): 'timestamp': 1477346700, 'upload_date': '20161024', 'duration': 2766, - 'creator': 'Third Ear Studio', + 'creators': ['Third Ear Studio'], 'series': 'Spår', 'episode': '2. Raggarmordet - Röster ur det förflutna', 'thumbnail': 'https://assets.pippa.io/shows/616ebe1886d7b1398620b943/616ebe33c7e6e70013cae7da.jpg', @@ -67,13 +67,16 @@ class ACastIE(ACastBaseIE): 'display_id': '2.raggarmordet-rosterurdetforflutna', 'season_number': 4, 'season': 'Season 4', - } + }, }, { 'url': 'http://embed.acast.com/adambuxton/ep.12-adam-joeschristmaspodcast2015', 'only_matching': True, }, { 'url': 'https://play.acast.com/s/rattegangspodden/s04e09styckmordetihelenelund-del2-2', 'only_matching': True, + }, { + 'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna', + 'only_matching': True, }, { 'url': 'https://play.acast.com/s/sparpodcast/2a92b283-1a75-4ad8-8396-499c641de0d9', 'only_matching': True, @@ -93,13 +96,13 @@ class ACastIE(ACastBaseIE): 'series': 'Democracy Sausage with Mark Kenny', 'timestamp': 1684826362, 'description': 'md5:feabe1fc5004c78ee59c84a46bf4ba16', - } + }, }] def _real_extract(self, url): channel, display_id = self._match_valid_url(url).groups() episode = self._call_api( - '%s/episodes/%s' % (channel, display_id), + f'{channel}/episodes/{display_id}', display_id, {'showInfo': 'true'}) return self._extract_episode( episode, self._extract_show_info(episode.get('show') or {})) @@ -110,7 +113,7 @@ class ACastChannelIE(ACastBaseIE): _VALID_URL = r'''(?x) https?:// (?: - (?:www\.)?acast\.com/| + (?:(?:www|shows)\.)?acast\.com/| play\.acast\.com/s/ ) (?P[^/#?]+) @@ -120,17 +123,20 @@ class ACastChannelIE(ACastBaseIE): 'info_dict': { 'id': '4efc5294-5385-4847-98bd-519799ce5786', 'title': 'Today in Focus', - 'description': 'md5:c09ce28c91002ce4ffce71d6504abaae', + 'description': 'md5:feca253de9947634605080cd9eeea2bf', }, 'playlist_mincount': 200, }, { 'url': 'http://play.acast.com/s/ft-banking-weekly', 'only_matching': True, + }, { + 'url': 'https://shows.acast.com/sparpodcast', + 'only_matching': True, }] @classmethod def suitable(cls, url): - return False if ACastIE.suitable(url) else super(ACastChannelIE, cls).suitable(url) + return False if ACastIE.suitable(url) else super().suitable(url) def _real_extract(self, url): show_slug = self._match_id(url) diff --git a/plugins/youtube_download/yt_dlp/extractor/acfun.py b/plugins/youtube_download/yt_dlp/extractor/acfun.py index dc579294..28559bae 100644 --- a/plugins/youtube_download/yt_dlp/extractor/acfun.py +++ b/plugins/youtube_download/yt_dlp/extractor/acfun.py @@ -3,9 +3,10 @@ from ..utils import ( float_or_none, format_field, int_or_none, - traverse_obj, parse_codecs, parse_qs, + str_or_none, + traverse_obj, ) @@ -24,7 +25,7 @@ class AcFunVideoBaseIE(InfoExtractor): 'width': int_or_none(video.get('width')), 'height': int_or_none(video.get('height')), 'tbr': float_or_none(video.get('avgBitrate')), - **parse_codecs(video.get('codecs', '')) + **parse_codecs(video.get('codecs', '')), }) return { @@ -76,7 +77,7 @@ class AcFunVideoIE(AcFunVideoBaseIE): 'comment_count': int, 'thumbnail': r're:^https?://.*\.(jpg|jpeg)', 'description': 'md5:67583aaf3a0f933bd606bc8a2d3ebb17', - } + }, }] def _real_extract(self, url): @@ -129,7 +130,7 @@ class AcFunBangumiIE(AcFunVideoBaseIE): 'title': '红孩儿之趴趴蛙寻石记 第5话 ', 'duration': 760.0, 'season': '红孩儿之趴趴蛙寻石记', - 'season_id': 5023171, + 'season_id': '5023171', 'season_number': 1, # series has only 1 season 'episode': 'Episode 5', 'episode_number': 5, @@ -146,7 +147,7 @@ class AcFunBangumiIE(AcFunVideoBaseIE): 'title': '叽歪老表（第二季） 第5话 坚不可摧', 'season': '叽歪老表（第二季）', 'season_number': 2, - 'season_id': 6065485, + 'season_id': '6065485', 'episode': '坚不可摧', 'episode_number': 5, 'upload_date': '20220324', @@ -191,7 +192,7 @@ class AcFunBangumiIE(AcFunVideoBaseIE): 'title': json_bangumi_data.get('showTitle'), 'thumbnail': json_bangumi_data.get('image'), 'season': json_bangumi_data.get('bangumiTitle'), - 'season_id': season_id, + 'season_id': str_or_none(season_id), 'season_number': season_number, 'episode': json_bangumi_data.get('title'), 'episode_number': episode_number, diff --git a/plugins/youtube_download/yt_dlp/extractor/adn.py b/plugins/youtube_download/yt_dlp/extractor/adn.py index b59dbc85..7dff4055 100644 --- a/plugins/youtube_download/yt_dlp/extractor/adn.py +++ b/plugins/youtube_download/yt_dlp/extractor/adn.py @@ -3,33 +3,53 @@ import binascii import json import os import random +import time from .common import InfoExtractor from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7 -from ..compat import compat_b64decode from ..networking.exceptions import HTTPError from ..utils import ( - ass_subtitles_timecode, - bytes_to_intlist, - bytes_to_long, ExtractorError, + ass_subtitles_timecode, + bytes_to_long, float_or_none, int_or_none, - intlist_to_bytes, + join_nonempty, long_to_bytes, + parse_iso8601, pkcs1pad, + str_or_none, strip_or_none, try_get, unified_strdate, urlencode_postdata, ) +from ..utils.traversal import traverse_obj -class ADNIE(InfoExtractor): +class ADNBaseIE(InfoExtractor): IE_DESC = 'Animation Digital Network' - _VALID_URL = r'https?://(?:www\.)?(?:animation|anime)digitalnetwork\.fr/video/[^/]+/(?P\d+)' + _NETRC_MACHINE = 'animationdigitalnetwork' + _BASE = 'animationdigitalnetwork.fr' + _API_BASE_URL = f'https://gw.api.{_BASE}/' + _PLAYER_BASE_URL = f'{_API_BASE_URL}player/' + _HEADERS = {} + _LOGIN_ERR_MESSAGE = 'Unable to log in' + _RSA_KEY = (0x9B42B08905199A5CCE2026274399CA560ECB209EE9878A708B1C0812E1BB8CB5D1FB7441861147C1A1F2F3A0476DD63A9CAC20D3E983613346850AA6CB38F16DC7D720FD7D86FC6E5B3D5BBC72E14CD0BF9E869F2CEA2CCAD648F1DCE38F1FF916CEFB2D339B64AA0264372344BC775E265E8A852F88144AB0BD9AA06C1A4ABB, 65537) + _POS_ALIGN_MAP = { + 'start': 1, + 'end': 3, + } + _LINE_ALIGN_MAP = { + 'middle': 8, + 'end': 4, + } + + +class ADNIE(ADNBaseIE): + _VALID_URL = r'https?://(?:www\.)?animationdigitalnetwork\.com/(?:(?Pde)/)?video/[^/?#]+/(?P\d+)' _TESTS = [{ - 'url': 'https://animationdigitalnetwork.fr/video/fruits-basket/9841-episode-1-a-ce-soir', + 'url': 'https://animationdigitalnetwork.com/video/558-fruits-basket/9841-episode-1-a-ce-soir', 'md5': '1c9ef066ceb302c86f80c2b371615261', 'info_dict': { 'id': '9841', @@ -44,29 +64,32 @@ class ADNIE(InfoExtractor): 'season_number': 1, 'episode': 'À ce soir !', 'episode_number': 1, + 'thumbnail': str, + 'season': 'Season 1', }, - 'skip': 'Only available in region (FR, ...)', + 'skip': 'Only available in French and German speaking Europe', }, { - 'url': 'http://animedigitalnetwork.fr/video/blue-exorcist-kyoto-saga/7778-episode-1-debut-des-hostilites', - 'only_matching': True, + 'url': 'https://animationdigitalnetwork.com/de/video/973-the-eminence-in-shadow/23550-folge-1', + 'md5': '5c5651bf5791fa6fcd7906012b9d94e8', + 'info_dict': { + 'id': '23550', + 'ext': 'mp4', + 'episode_number': 1, + 'duration': 1417, + 'release_date': '20231004', + 'series': 'The Eminence in Shadow', + 'season_number': 2, + 'episode': str, + 'title': str, + 'thumbnail': str, + 'season': 'Season 2', + 'comment_count': int, + 'average_rating': float, + 'description': str, + }, + # 'skip': 'Only available in French and German speaking Europe', }] - _NETRC_MACHINE = 'animationdigitalnetwork' - _BASE = 'animationdigitalnetwork.fr' - _API_BASE_URL = 'https://gw.api.' + _BASE + '/' - _PLAYER_BASE_URL = _API_BASE_URL + 'player/' - _HEADERS = {} - _LOGIN_ERR_MESSAGE = 'Unable to log in' - _RSA_KEY = (0x9B42B08905199A5CCE2026274399CA560ECB209EE9878A708B1C0812E1BB8CB5D1FB7441861147C1A1F2F3A0476DD63A9CAC20D3E983613346850AA6CB38F16DC7D720FD7D86FC6E5B3D5BBC72E14CD0BF9E869F2CEA2CCAD648F1DCE38F1FF916CEFB2D339B64AA0264372344BC775E265E8A852F88144AB0BD9AA06C1A4ABB, 65537) - _POS_ALIGN_MAP = { - 'start': 1, - 'end': 3, - } - _LINE_ALIGN_MAP = { - 'middle': 8, - 'end': 4, - } - def _get_subtitles(self, sub_url, video_id): if not sub_url: return None @@ -83,9 +106,9 @@ class ADNIE(InfoExtractor): # http://animationdigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js dec_subtitles = unpad_pkcs7(aes_cbc_decrypt_bytes( - compat_b64decode(enc_subtitles[24:]), + base64.b64decode(enc_subtitles[24:]), binascii.unhexlify(self._K + '7fac1178830cfe0c'), - compat_b64decode(enc_subtitles[:24]))) + base64.b64decode(enc_subtitles[:24]))) subtitles_json = self._parse_json(dec_subtitles.decode(), None, fatal=False) if not subtitles_json: return None @@ -108,7 +131,7 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text''' if start is None or end is None or text is None: continue alignment = self._POS_ALIGN_MAP.get(position_align, 2) + self._LINE_ALIGN_MAP.get(line_align, 0) - ssa += os.linesep + 'Dialogue: Marked=0,%s,%s,Default,,0,0,0,,%s%s' % ( + ssa += os.linesep + 'Dialogue: Marked=0,{},{},Default,,0,0,0,,{}{}'.format( ass_subtitles_timecode(start), ass_subtitles_timecode(end), '{\\a%d}' % alignment if alignment != 2 else '', @@ -116,6 +139,8 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text''' if sub_lang == 'vostf': sub_lang = 'fr' + elif sub_lang == 'vostde': + sub_lang = 'de' subtitles.setdefault(sub_lang, []).extend([{ 'ext': 'json', 'data': json.dumps(sub), @@ -137,7 +162,7 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text''' 'username': username, })) or {}).get('accessToken') if access_token: - self._HEADERS = {'authorization': 'Bearer ' + access_token} + self._HEADERS['Authorization'] = f'Bearer {access_token}' except ExtractorError as e: message = None if isinstance(e.cause, HTTPError) and e.cause.status == 401: @@ -147,8 +172,9 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text''' self.report_warning(message or self._LOGIN_ERR_MESSAGE) def _real_extract(self, url): - video_id = self._match_id(url) - video_base_url = self._PLAYER_BASE_URL + 'video/%s/' % video_id + lang, video_id = self._match_valid_url(url).group('lang', 'id') + self._HEADERS['X-Target-Distribution'] = lang or 'fr' + video_base_url = self._PLAYER_BASE_URL + f'video/{video_id}/' player = self._download_json( video_base_url + 'configuration', video_id, 'Downloading player config JSON metadata', @@ -157,26 +183,29 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text''' user = options['user'] if not user.get('hasAccess'): - self.raise_login_required() + start_date = traverse_obj(options, ('video', 'startDate', {str})) + if (parse_iso8601(start_date) or 0) > time.time(): + raise ExtractorError(f'This video is not available yet. Release date: {start_date}', expected=True) + self.raise_login_required('This video requires a subscription', method='password') token = self._download_json( user.get('refreshTokenUrl') or (self._PLAYER_BASE_URL + 'refresh/token'), video_id, 'Downloading access token', headers={ - 'x-player-refresh-token': user['refreshToken'] + 'X-Player-Refresh-Token': user['refreshToken'], }, data=b'')['token'] links_url = try_get(options, lambda x: x['video']['url']) or (video_base_url + 'link') self._K = ''.join(random.choices('0123456789abcdef', k=16)) - message = bytes_to_intlist(json.dumps({ + message = list(json.dumps({ 'k': self._K, 't': token, - })) + }).encode()) # Sometimes authentication fails for no good reason, retry with # a different random padding links_data = None for _ in range(3): - padded_message = intlist_to_bytes(pkcs1pad(message, 128)) + padded_message = bytes(pkcs1pad(message, 128)) n, e = self._RSA_KEY encrypted_message = long_to_bytes(pow(bytes_to_long(padded_message), e, n)) authorization = base64.b64encode(encrypted_message).decode() @@ -184,12 +213,13 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text''' try: links_data = self._download_json( links_url, video_id, 'Downloading links JSON metadata', headers={ - 'X-Player-Token': authorization + 'X-Player-Token': authorization, + **self._HEADERS, }, query={ 'freeWithAds': 'true', 'adaptive': 'false', 'withMetadata': 'true', - 'source': 'Web' + 'source': 'Web', }) break except ExtractorError as e: @@ -202,7 +232,7 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text''' error = self._parse_json(e.cause.response.read(), video_id) message = error.get('message') - if e.cause.code == 403 and error.get('code') == 'player-bad-geolocation-country': + if e.cause.status == 403 and error.get('code') == 'player-bad-geolocation-country': self.raise_geo_restricted(msg=message) raise ExtractorError(message) else: @@ -221,7 +251,8 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text''' for quality, load_balancer_url in qualities.items(): load_balancer_data = self._download_json( load_balancer_url, video_id, - 'Downloading %s %s JSON metadata' % (format_id, quality), + f'Downloading {format_id} {quality} JSON metadata', + headers=self._HEADERS, fatal=False) or {} m3u8_url = load_balancer_data.get('location') if not m3u8_url: @@ -232,11 +263,17 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text''' if format_id == 'vf': for f in m3u8_formats: f['language'] = 'fr' + elif format_id == 'vde': + for f in m3u8_formats: + f['language'] = 'de' formats.extend(m3u8_formats) + if not formats: + self.raise_login_required('This video requires a subscription', method='password') + video = (self._download_json( - self._API_BASE_URL + 'video/%s' % video_id, video_id, - 'Downloading additional video metadata', fatal=False) or {}).get('video') or {} + self._API_BASE_URL + f'video/{video_id}', video_id, + 'Downloading additional video metadata', fatal=False, headers=self._HEADERS) or {}).get('video') or {} show = video.get('show') or {} return { @@ -255,3 +292,38 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text''' 'average_rating': float_or_none(video.get('rating') or metas.get('rating')), 'comment_count': int_or_none(video.get('commentsCount')), } + + +class ADNSeasonIE(ADNBaseIE): + _VALID_URL = r'https?://(?:www\.)?animationdigitalnetwork\.com/(?:(?Pde)/)?video/(?P\d+)[^/?#]*/?(?:$|[#?])' + _TESTS = [{ + 'url': 'https://animationdigitalnetwork.com/video/911-tokyo-mew-mew-new', + 'playlist_count': 12, + 'info_dict': { + 'id': '911', + 'title': 'Tokyo Mew Mew New', + }, + # 'skip': 'Only available in French end German speaking Europe', + }] + + def _real_extract(self, url): + lang, video_show_slug = self._match_valid_url(url).group('lang', 'id') + self._HEADERS['X-Target-Distribution'] = lang or 'fr' + show = self._download_json( + f'{self._API_BASE_URL}show/{video_show_slug}/', video_show_slug, + 'Downloading show JSON metadata', headers=self._HEADERS)['show'] + show_id = str(show['id']) + episodes = self._download_json( + f'{self._API_BASE_URL}video/show/{show_id}', video_show_slug, + 'Downloading episode list', headers=self._HEADERS, query={ + 'order': 'asc', + 'limit': '-1', + }) + + def entries(): + for episode_id in traverse_obj(episodes, ('videos', ..., 'id', {str_or_none})): + yield self.url_result(join_nonempty( + 'https://animationdigitalnetwork.com', lang, 'video', + video_show_slug, episode_id, delim='/'), ADNIE, episode_id) + + return self.playlist_result(entries(), show_id, show.get('title')) diff --git a/plugins/youtube_download/yt_dlp/extractor/adobeconnect.py b/plugins/youtube_download/yt_dlp/extractor/adobeconnect.py index 8963b128..b2934d6f 100644 --- a/plugins/youtube_download/yt_dlp/extractor/adobeconnect.py +++ b/plugins/youtube_download/yt_dlp/extractor/adobeconnect.py @@ -1,8 +1,6 @@ +import urllib.parse + from .common import InfoExtractor -from ..compat import ( - compat_parse_qs, - compat_urlparse, -) class AdobeConnectIE(InfoExtractor): @@ -12,13 +10,13 @@ class AdobeConnectIE(InfoExtractor): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) title = self._html_extract_title(webpage) - qs = compat_parse_qs(self._search_regex(r"swfUrl\s*=\s*'([^']+)'", webpage, 'swf url').split('?')[1]) + qs = urllib.parse.parse_qs(self._search_regex(r"swfUrl\s*=\s*'([^']+)'", webpage, 'swf url').split('?')[1]) is_live = qs.get('isLive', ['false'])[0] == 'true' formats = [] for con_string in qs['conStrings'][0].split(','): formats.append({ 'format_id': con_string.split('://')[0], - 'app': compat_urlparse.quote('?' + con_string.split('?')[1] + 'flvplayerapp/' + qs['appInstance'][0]), + 'app': urllib.parse.quote('?' + con_string.split('?')[1] + 'flvplayerapp/' + qs['appInstance'][0]), 'ext': 'flv', 'play_path': 'mp4:' + qs['streamName'][0], 'rtmp_conn': 'S:' + qs['ticket'][0], diff --git a/plugins/youtube_download/yt_dlp/extractor/adobepass.py b/plugins/youtube_download/yt_dlp/extractor/adobepass.py index 5eed0ca2..f1b87792 100644 --- a/plugins/youtube_download/yt_dlp/extractor/adobepass.py +++ b/plugins/youtube_download/yt_dlp/extractor/adobepass.py @@ -2,10 +2,10 @@ import getpass import json import re import time +import urllib.parse import xml.etree.ElementTree as etree from .common import InfoExtractor -from ..compat import compat_urlparse from ..networking.exceptions import HTTPError from ..utils import ( NO_DEFAULT, @@ -68,7 +68,7 @@ MSO_INFO = { }, 'Philo': { 'name': 'Philo', - 'username_field': 'ident' + 'username_field': 'ident', }, 'Verizon': { 'name': 'Verizon FiOS', @@ -81,1258 +81,1258 @@ MSO_INFO = { 'password_field': 'j_password', }, 'thr030': { - 'name': '3 Rivers Communications' + 'name': '3 Rivers Communications', }, 'com140': { - 'name': 'Access Montana' + 'name': 'Access Montana', }, 'acecommunications': { - 'name': 'AcenTek' + 'name': 'AcenTek', }, 'acm010': { - 'name': 'Acme Communications' + 'name': 'Acme Communications', }, 'ada020': { - 'name': 'Adams Cable Service' + 'name': 'Adams Cable Service', }, 'alb020': { - 'name': 'Albany Mutual Telephone' + 'name': 'Albany Mutual Telephone', }, 'algona': { - 'name': 'Algona Municipal Utilities' + 'name': 'Algona Municipal Utilities', }, 'allwest': { - 'name': 'All West Communications' + 'name': 'All West Communications', }, 'all025': { - 'name': 'Allen\'s Communications' + 'name': 'Allen\'s Communications', }, 'spl010': { - 'name': 'Alliance Communications' + 'name': 'Alliance Communications', }, 'all070': { - 'name': 'ALLO Communications' + 'name': 'ALLO Communications', }, 'alpine': { - 'name': 'Alpine Communications' + 'name': 'Alpine Communications', }, 'hun015': { - 'name': 'American Broadband' + 'name': 'American Broadband', }, 'nwc010': { - 'name': 'American Broadband Missouri' + 'name': 'American Broadband Missouri', }, 'com130-02': { - 'name': 'American Community Networks' + 'name': 'American Community Networks', }, 'com130-01': { - 'name': 'American Warrior Networks' + 'name': 'American Warrior Networks', }, 'tom020': { - 'name': 'Amherst Telephone/Tomorrow Valley' + 'name': 'Amherst Telephone/Tomorrow Valley', }, 'tvc020': { - 'name': 'Andycable' + 'name': 'Andycable', }, 'arkwest': { - 'name': 'Arkwest Communications' + 'name': 'Arkwest Communications', }, 'art030': { - 'name': 'Arthur Mutual Telephone Company' + 'name': 'Arthur Mutual Telephone Company', }, 'arvig': { - 'name': 'Arvig' + 'name': 'Arvig', }, 'nttcash010': { - 'name': 'Ashland Home Net' + 'name': 'Ashland Home Net', }, 'astound': { - 'name': 'Astound (now Wave)' + 'name': 'Astound (now Wave)', }, 'dix030': { - 'name': 'ATC Broadband' + 'name': 'ATC Broadband', }, 'ara010': { - 'name': 'ATC Communications' + 'name': 'ATC Communications', }, 'she030-02': { - 'name': 'Ayersville Communications' + 'name': 'Ayersville Communications', }, 'baldwin': { - 'name': 'Baldwin Lightstream' + 'name': 'Baldwin Lightstream', }, 'bal040': { - 'name': 'Ballard TV' + 'name': 'Ballard TV', }, 'cit025': { - 'name': 'Bardstown Cable TV' + 'name': 'Bardstown Cable TV', }, 'bay030': { - 'name': 'Bay Country Communications' + 'name': 'Bay Country Communications', }, 'tel095': { - 'name': 'Beaver Creek Cooperative Telephone' + 'name': 'Beaver Creek Cooperative Telephone', }, 'bea020': { - 'name': 'Beaver Valley Cable' + 'name': 'Beaver Valley Cable', }, 'bee010': { - 'name': 'Bee Line Cable' + 'name': 'Bee Line Cable', }, 'wir030': { - 'name': 'Beehive Broadband' + 'name': 'Beehive Broadband', }, 'bra020': { - 'name': 'BELD' + 'name': 'BELD', }, 'bel020': { - 'name': 'Bellevue Municipal Cable' + 'name': 'Bellevue Municipal Cable', }, 'vol040-01': { - 'name': 'Ben Lomand Connect / BLTV' + 'name': 'Ben Lomand Connect / BLTV', }, 'bev010': { - 'name': 'BEVCOMM' + 'name': 'BEVCOMM', }, 'big020': { - 'name': 'Big Sandy Broadband' + 'name': 'Big Sandy Broadband', }, 'ble020': { - 'name': 'Bledsoe Telephone Cooperative' + 'name': 'Bledsoe Telephone Cooperative', }, 'bvt010': { - 'name': 'Blue Valley Tele-Communications' + 'name': 'Blue Valley Tele-Communications', }, 'bra050': { - 'name': 'Brandenburg Telephone Co.' + 'name': 'Brandenburg Telephone Co.', }, 'bte010': { - 'name': 'Bristol Tennessee Essential Services' + 'name': 'Bristol Tennessee Essential Services', }, 'annearundel': { - 'name': 'Broadstripe' + 'name': 'Broadstripe', }, 'btc010': { - 'name': 'BTC Communications' + 'name': 'BTC Communications', }, 'btc040': { - 'name': 'BTC Vision - Nahunta' + 'name': 'BTC Vision - Nahunta', }, 'bul010': { - 'name': 'Bulloch Telephone Cooperative' + 'name': 'Bulloch Telephone Cooperative', }, 'but010': { - 'name': 'Butler-Bremer Communications' + 'name': 'Butler-Bremer Communications', }, 'tel160-csp': { - 'name': 'C Spire SNAP' + 'name': 'C Spire SNAP', }, 'csicable': { - 'name': 'Cable Services Inc.' + 'name': 'Cable Services Inc.', }, 'cableamerica': { - 'name': 'CableAmerica' + 'name': 'CableAmerica', }, 'cab038': { - 'name': 'CableSouth Media 3' + 'name': 'CableSouth Media 3', }, 'weh010-camtel': { - 'name': 'Cam-Tel Company' + 'name': 'Cam-Tel Company', }, 'car030': { - 'name': 'Cameron Communications' + 'name': 'Cameron Communications', }, 'canbytel': { - 'name': 'Canby Telcom' + 'name': 'Canby Telcom', }, 'crt020': { - 'name': 'CapRock Tv' + 'name': 'CapRock Tv', }, 'car050': { - 'name': 'Carnegie Cable' + 'name': 'Carnegie Cable', }, 'cas': { - 'name': 'CAS Cable' + 'name': 'CAS Cable', }, 'casscomm': { - 'name': 'CASSCOMM' + 'name': 'CASSCOMM', }, 'mid180-02': { - 'name': 'Catalina Broadband Solutions' + 'name': 'Catalina Broadband Solutions', }, 'cccomm': { - 'name': 'CC Communications' + 'name': 'CC Communications', }, 'nttccde010': { - 'name': 'CDE Lightband' + 'name': 'CDE Lightband', }, 'cfunet': { - 'name': 'Cedar Falls Utilities' + 'name': 'Cedar Falls Utilities', }, 'dem010-01': { - 'name': 'Celect-Bloomer Telephone Area' + 'name': 'Celect-Bloomer Telephone Area', }, 'dem010-02': { - 'name': 'Celect-Bruce Telephone Area' + 'name': 'Celect-Bruce Telephone Area', }, 'dem010-03': { - 'name': 'Celect-Citizens Connected Area' + 'name': 'Celect-Citizens Connected Area', }, 'dem010-04': { - 'name': 'Celect-Elmwood/Spring Valley Area' + 'name': 'Celect-Elmwood/Spring Valley Area', }, 'dem010-06': { - 'name': 'Celect-Mosaic Telecom' + 'name': 'Celect-Mosaic Telecom', }, 'dem010-05': { - 'name': 'Celect-West WI Telephone Area' + 'name': 'Celect-West WI Telephone Area', }, 'net010-02': { - 'name': 'Cellcom/Nsight Telservices' + 'name': 'Cellcom/Nsight Telservices', }, 'cen100': { - 'name': 'CentraCom' + 'name': 'CentraCom', }, 'nttccst010': { - 'name': 'Central Scott / CSTV' + 'name': 'Central Scott / CSTV', }, 'cha035': { - 'name': 'Chaparral CableVision' + 'name': 'Chaparral CableVision', }, 'cha050': { - 'name': 'Chariton Valley Communication Corporation, Inc.' + 'name': 'Chariton Valley Communication Corporation, Inc.', }, 'cha060': { - 'name': 'Chatmoss Cablevision' + 'name': 'Chatmoss Cablevision', }, 'nttcche010': { - 'name': 'Cherokee Communications' + 'name': 'Cherokee Communications', }, 'che050': { - 'name': 'Chesapeake Bay Communications' + 'name': 'Chesapeake Bay Communications', }, 'cimtel': { - 'name': 'Cim-Tel Cable, LLC.' + 'name': 'Cim-Tel Cable, LLC.', }, 'cit180': { - 'name': 'Citizens Cablevision - Floyd, VA' + 'name': 'Citizens Cablevision - Floyd, VA', }, 'cit210': { - 'name': 'Citizens Cablevision, Inc.' + 'name': 'Citizens Cablevision, Inc.', }, 'cit040': { - 'name': 'Citizens Fiber' + 'name': 'Citizens Fiber', }, 'cit250': { - 'name': 'Citizens Mutual' + 'name': 'Citizens Mutual', }, 'war040': { - 'name': 'Citizens Telephone Corporation' + 'name': 'Citizens Telephone Corporation', }, 'wat025': { - 'name': 'City Of Monroe' + 'name': 'City Of Monroe', }, 'wadsworth': { - 'name': 'CityLink' + 'name': 'CityLink', }, 'nor100': { - 'name': 'CL Tel' + 'name': 'CL Tel', }, 'cla010': { - 'name': 'Clarence Telephone and Cedar Communications' + 'name': 'Clarence Telephone and Cedar Communications', }, 'ser060': { - 'name': 'Clear Choice Communications' + 'name': 'Clear Choice Communications', }, 'tac020': { - 'name': 'Click! Cable TV' + 'name': 'Click! Cable TV', }, 'war020': { - 'name': 'CLICK1.NET' + 'name': 'CLICK1.NET', }, 'cml010': { - 'name': 'CML Telephone Cooperative Association' + 'name': 'CML Telephone Cooperative Association', }, 'cns': { - 'name': 'CNS' + 'name': 'CNS', }, 'com160': { - 'name': 'Co-Mo Connect' + 'name': 'Co-Mo Connect', }, 'coa020': { - 'name': 'Coast Communications' + 'name': 'Coast Communications', }, 'coa030': { - 'name': 'Coaxial Cable TV' + 'name': 'Coaxial Cable TV', }, 'mid055': { - 'name': 'Cobalt TV (Mid-State Community TV)' + 'name': 'Cobalt TV (Mid-State Community TV)', }, 'col070': { - 'name': 'Columbia Power & Water Systems' + 'name': 'Columbia Power & Water Systems', }, 'col080': { - 'name': 'Columbus Telephone' + 'name': 'Columbus Telephone', }, 'nor105': { - 'name': 'Communications 1 Cablevision, Inc.' + 'name': 'Communications 1 Cablevision, Inc.', }, 'com150': { - 'name': 'Community Cable & Broadband' + 'name': 'Community Cable & Broadband', }, 'com020': { - 'name': 'Community Communications Company' + 'name': 'Community Communications Company', }, 'coy010': { - 'name': 'commZoom' + 'name': 'commZoom', }, 'com025': { - 'name': 'Complete Communication Services' + 'name': 'Complete Communication Services', }, 'cat020': { - 'name': 'Comporium' + 'name': 'Comporium', }, 'com071': { - 'name': 'ComSouth Telesys' + 'name': 'ComSouth Telesys', }, 'consolidatedcable': { - 'name': 'Consolidated' + 'name': 'Consolidated', }, 'conwaycorp': { - 'name': 'Conway Corporation' + 'name': 'Conway Corporation', }, 'coo050': { - 'name': 'Coon Valley Telecommunications Inc' + 'name': 'Coon Valley Telecommunications Inc', }, 'coo080': { - 'name': 'Cooperative Telephone Company' + 'name': 'Cooperative Telephone Company', }, 'cpt010': { - 'name': 'CP-TEL' + 'name': 'CP-TEL', }, 'cra010': { - 'name': 'Craw-Kan Telephone' + 'name': 'Craw-Kan Telephone', }, 'crestview': { - 'name': 'Crestview Cable Communications' + 'name': 'Crestview Cable Communications', }, 'cross': { - 'name': 'Cross TV' + 'name': 'Cross TV', }, 'cro030': { - 'name': 'Crosslake Communications' + 'name': 'Crosslake Communications', }, 'ctc040': { - 'name': 'CTC - Brainerd MN' + 'name': 'CTC - Brainerd MN', }, 'phe030': { - 'name': 'CTV-Beam - East Alabama' + 'name': 'CTV-Beam - East Alabama', }, 'cun010': { - 'name': 'Cunningham Telephone & Cable' + 'name': 'Cunningham Telephone & Cable', }, 'dpc010': { - 'name': 'D & P Communications' + 'name': 'D & P Communications', }, 'dak030': { - 'name': 'Dakota Central Telecommunications' + 'name': 'Dakota Central Telecommunications', }, 'nttcdel010': { - 'name': 'Delcambre Telephone LLC' + 'name': 'Delcambre Telephone LLC', }, 'tel160-del': { - 'name': 'Delta Telephone Company' + 'name': 'Delta Telephone Company', }, 'sal040': { - 'name': 'DiamondNet' + 'name': 'DiamondNet', }, 'ind060-dc': { - 'name': 'Direct Communications' + 'name': 'Direct Communications', }, 'doy010': { - 'name': 'Doylestown Cable TV' + 'name': 'Doylestown Cable TV', }, 'dic010': { - 'name': 'DRN' + 'name': 'DRN', }, 'dtc020': { - 'name': 'DTC' + 'name': 'DTC', }, 'dtc010': { - 'name': 'DTC Cable (Delhi)' + 'name': 'DTC Cable (Delhi)', }, 'dum010': { - 'name': 'Dumont Telephone Company' + 'name': 'Dumont Telephone Company', }, 'dun010': { - 'name': 'Dunkerton Telephone Cooperative' + 'name': 'Dunkerton Telephone Cooperative', }, 'cci010': { - 'name': 'Duo County Telecom' + 'name': 'Duo County Telecom', }, 'eagle': { - 'name': 'Eagle Communications' + 'name': 'Eagle Communications', }, 'weh010-east': { - 'name': 'East Arkansas Cable TV' + 'name': 'East Arkansas Cable TV', }, 'eatel': { - 'name': 'EATEL Video, LLC' + 'name': 'EATEL Video, LLC', }, 'ell010': { - 'name': 'ECTA' + 'name': 'ECTA', }, 'emerytelcom': { - 'name': 'Emery Telcom Video LLC' + 'name': 'Emery Telcom Video LLC', }, 'nor200': { - 'name': 'Empire Access' + 'name': 'Empire Access', }, 'endeavor': { - 'name': 'Endeavor Communications' + 'name': 'Endeavor Communications', }, 'sun045': { - 'name': 'Enhanced Telecommunications Corporation' + 'name': 'Enhanced Telecommunications Corporation', }, 'mid030': { - 'name': 'enTouch' + 'name': 'enTouch', }, 'epb020': { - 'name': 'EPB Smartnet' + 'name': 'EPB Smartnet', }, 'jea010': { - 'name': 'EPlus Broadband' + 'name': 'EPlus Broadband', }, 'com065': { - 'name': 'ETC' + 'name': 'ETC', }, 'ete010': { - 'name': 'Etex Communications' + 'name': 'Etex Communications', }, 'fbc-tele': { - 'name': 'F&B Communications' + 'name': 'F&B Communications', }, 'fal010': { - 'name': 'Falcon Broadband' + 'name': 'Falcon Broadband', }, 'fam010': { - 'name': 'FamilyView CableVision' + 'name': 'FamilyView CableVision', }, 'far020': { - 'name': 'Farmers Mutual Telephone Company' + 'name': 'Farmers Mutual Telephone Company', }, 'fay010': { - 'name': 'Fayetteville Public Utilities' + 'name': 'Fayetteville Public Utilities', }, 'sal060': { - 'name': 'fibrant' + 'name': 'fibrant', }, 'fid010': { - 'name': 'Fidelity Communications' + 'name': 'Fidelity Communications', }, 'for030': { - 'name': 'FJ Communications' + 'name': 'FJ Communications', }, 'fli020': { - 'name': 'Flint River Communications' + 'name': 'Flint River Communications', }, 'far030': { - 'name': 'FMT - Jesup' + 'name': 'FMT - Jesup', }, 'foo010': { - 'name': 'Foothills Communications' + 'name': 'Foothills Communications', }, 'for080': { - 'name': 'Forsyth CableNet' + 'name': 'Forsyth CableNet', }, 'fbcomm': { - 'name': 'Frankfort Plant Board' + 'name': 'Frankfort Plant Board', }, 'tel160-fra': { - 'name': 'Franklin Telephone Company' + 'name': 'Franklin Telephone Company', }, 'nttcftc010': { - 'name': 'FTC' + 'name': 'FTC', }, 'fullchannel': { - 'name': 'Full Channel, Inc.' + 'name': 'Full Channel, Inc.', }, 'gar040': { - 'name': 'Gardonville Cooperative Telephone Association' + 'name': 'Gardonville Cooperative Telephone Association', }, 'gbt010': { - 'name': 'GBT Communications, Inc.' + 'name': 'GBT Communications, Inc.', }, 'tec010': { - 'name': 'Genuine Telecom' + 'name': 'Genuine Telecom', }, 'clr010': { - 'name': 'Giant Communications' + 'name': 'Giant Communications', }, 'gla010': { - 'name': 'Glasgow EPB' + 'name': 'Glasgow EPB', }, 'gle010': { - 'name': 'Glenwood Telecommunications' + 'name': 'Glenwood Telecommunications', }, 'gra060': { - 'name': 'GLW Broadband Inc.' + 'name': 'GLW Broadband Inc.', }, 'goldenwest': { - 'name': 'Golden West Cablevision' + 'name': 'Golden West Cablevision', }, 'vis030': { - 'name': 'Grantsburg Telcom' + 'name': 'Grantsburg Telcom', }, 'gpcom': { - 'name': 'Great Plains Communications' + 'name': 'Great Plains Communications', }, 'gri010': { - 'name': 'Gridley Cable Inc' + 'name': 'Gridley Cable Inc', }, 'hbc010': { - 'name': 'H&B Cable Services' + 'name': 'H&B Cable Services', }, 'hae010': { - 'name': 'Haefele TV Inc.' + 'name': 'Haefele TV Inc.', }, 'htc010': { - 'name': 'Halstad Telephone Company' + 'name': 'Halstad Telephone Company', }, 'har005': { - 'name': 'Harlan Municipal Utilities' + 'name': 'Harlan Municipal Utilities', }, 'har020': { - 'name': 'Hart Communications' + 'name': 'Hart Communications', }, 'ced010': { - 'name': 'Hartelco TV' + 'name': 'Hartelco TV', }, 'hea040': { - 'name': 'Heart of Iowa Communications Cooperative' + 'name': 'Heart of Iowa Communications Cooperative', }, 'htc020': { - 'name': 'Hickory Telephone Company' + 'name': 'Hickory Telephone Company', }, 'nttchig010': { - 'name': 'Highland Communication Services' + 'name': 'Highland Communication Services', }, 'hig030': { - 'name': 'Highland Media' + 'name': 'Highland Media', }, 'spc010': { - 'name': 'Hilliary Communications' + 'name': 'Hilliary Communications', }, 'hin020': { - 'name': 'Hinton CATV Co.' + 'name': 'Hinton CATV Co.', }, 'hometel': { - 'name': 'HomeTel Entertainment, Inc.' + 'name': 'HomeTel Entertainment, Inc.', }, 'hoodcanal': { - 'name': 'Hood Canal Communications' + 'name': 'Hood Canal Communications', }, 'weh010-hope': { - 'name': 'Hope - Prescott Cable TV' + 'name': 'Hope - Prescott Cable TV', }, 'horizoncable': { - 'name': 'Horizon Cable TV, Inc.' + 'name': 'Horizon Cable TV, Inc.', }, 'hor040': { - 'name': 'Horizon Chillicothe Telephone' + 'name': 'Horizon Chillicothe Telephone', }, 'htc030': { - 'name': 'HTC Communications Co. - IL' + 'name': 'HTC Communications Co. - IL', }, 'htccomm': { - 'name': 'HTC Communications, Inc. - IA' + 'name': 'HTC Communications, Inc. - IA', }, 'wal005': { - 'name': 'Huxley Communications' + 'name': 'Huxley Communications', }, 'imon': { - 'name': 'ImOn Communications' + 'name': 'ImOn Communications', }, 'ind040': { - 'name': 'Independence Telecommunications' + 'name': 'Independence Telecommunications', }, 'rrc010': { - 'name': 'Inland Networks' + 'name': 'Inland Networks', }, 'stc020': { - 'name': 'Innovative Cable TV St Croix' + 'name': 'Innovative Cable TV St Croix', }, 'car100': { - 'name': 'Innovative Cable TV St Thomas-St John' + 'name': 'Innovative Cable TV St Thomas-St John', }, 'icc010': { - 'name': 'Inside Connect Cable' + 'name': 'Inside Connect Cable', }, 'int100': { - 'name': 'Integra Telecom' + 'name': 'Integra Telecom', }, 'int050': { - 'name': 'Interstate Telecommunications Coop' + 'name': 'Interstate Telecommunications Coop', }, 'irv010': { - 'name': 'Irvine Cable' + 'name': 'Irvine Cable', }, 'k2c010': { - 'name': 'K2 Communications' + 'name': 'K2 Communications', }, 'kal010': { - 'name': 'Kalida Telephone Company, Inc.' + 'name': 'Kalida Telephone Company, Inc.', }, 'kal030': { - 'name': 'Kalona Cooperative Telephone Company' + 'name': 'Kalona Cooperative Telephone Company', }, 'kmt010': { - 'name': 'KMTelecom' + 'name': 'KMTelecom', }, 'kpu010': { - 'name': 'KPU Telecommunications' + 'name': 'KPU Telecommunications', }, 'kuh010': { - 'name': 'Kuhn Communications, Inc.' + 'name': 'Kuhn Communications, Inc.', }, 'lak130': { - 'name': 'Lakeland Communications' + 'name': 'Lakeland Communications', }, 'lan010': { - 'name': 'Langco' + 'name': 'Langco', }, 'lau020': { - 'name': 'Laurel Highland Total Communications, Inc.' + 'name': 'Laurel Highland Total Communications, Inc.', }, 'leh010': { - 'name': 'Lehigh Valley Cooperative Telephone' + 'name': 'Lehigh Valley Cooperative Telephone', }, 'bra010': { - 'name': 'Limestone Cable/Bracken Cable' + 'name': 'Limestone Cable/Bracken Cable', }, 'loc020': { - 'name': 'LISCO' + 'name': 'LISCO', }, 'lit020': { - 'name': 'Litestream' + 'name': 'Litestream', }, 'tel140': { - 'name': 'LivCom' + 'name': 'LivCom', }, 'loc010': { - 'name': 'LocalTel Communications' + 'name': 'LocalTel Communications', }, 'weh010-longview': { - 'name': 'Longview - Kilgore Cable TV' + 'name': 'Longview - Kilgore Cable TV', }, 'lon030': { - 'name': 'Lonsdale Video Ventures, LLC' + 'name': 'Lonsdale Video Ventures, LLC', }, 'lns010': { - 'name': 'Lost Nation-Elwood Telephone Co.' + 'name': 'Lost Nation-Elwood Telephone Co.', }, 'nttclpc010': { - 'name': 'LPC Connect' + 'name': 'LPC Connect', }, 'lumos': { - 'name': 'Lumos Networks' + 'name': 'Lumos Networks', }, 'madison': { - 'name': 'Madison Communications' + 'name': 'Madison Communications', }, 'mad030': { - 'name': 'Madison County Cable Inc.' + 'name': 'Madison County Cable Inc.', }, 'nttcmah010': { - 'name': 'Mahaska Communication Group' + 'name': 'Mahaska Communication Group', }, 'mar010': { - 'name': 'Marne & Elk Horn Telephone Company' + 'name': 'Marne & Elk Horn Telephone Company', }, 'mcc040': { - 'name': 'McClure Telephone Co.' + 'name': 'McClure Telephone Co.', }, 'mctv': { - 'name': 'MCTV' + 'name': 'MCTV', }, 'merrimac': { - 'name': 'Merrimac Communications Ltd.' + 'name': 'Merrimac Communications Ltd.', }, 'metronet': { - 'name': 'Metronet' + 'name': 'Metronet', }, 'mhtc': { - 'name': 'MHTC' + 'name': 'MHTC', }, 'midhudson': { - 'name': 'Mid-Hudson Cable' + 'name': 'Mid-Hudson Cable', }, 'midrivers': { - 'name': 'Mid-Rivers Communications' + 'name': 'Mid-Rivers Communications', }, 'mid045': { - 'name': 'Midstate Communications' + 'name': 'Midstate Communications', }, 'mil080': { - 'name': 'Milford Communications' + 'name': 'Milford Communications', }, 'min030': { - 'name': 'MINET' + 'name': 'MINET', }, 'nttcmin010': { - 'name': 'Minford TV' + 'name': 'Minford TV', }, 'san040-02': { - 'name': 'Mitchell Telecom' + 'name': 'Mitchell Telecom', }, 'mlg010': { - 'name': 'MLGC' + 'name': 'MLGC', }, 'mon060': { - 'name': 'Mon-Cre TVE' + 'name': 'Mon-Cre TVE', }, 'mou110': { - 'name': 'Mountain Telephone' + 'name': 'Mountain Telephone', }, 'mou050': { - 'name': 'Mountain Village Cable' + 'name': 'Mountain Village Cable', }, 'mtacomm': { - 'name': 'MTA Communications, LLC' + 'name': 'MTA Communications, LLC', }, 'mtc010': { - 'name': 'MTC Cable' + 'name': 'MTC Cable', }, 'med040': { - 'name': 'MTC Technologies' + 'name': 'MTC Technologies', }, 'man060': { - 'name': 'MTCC' + 'name': 'MTCC', }, 'mtc030': { - 'name': 'MTCO Communications' + 'name': 'MTCO Communications', }, 'mul050': { - 'name': 'Mulberry Telecommunications' + 'name': 'Mulberry Telecommunications', }, 'mur010': { - 'name': 'Murray Electric System' + 'name': 'Murray Electric System', }, 'musfiber': { - 'name': 'MUS FiberNET' + 'name': 'MUS FiberNET', }, 'mpw': { - 'name': 'Muscatine Power & Water' + 'name': 'Muscatine Power & Water', }, 'nttcsli010': { - 'name': 'myEVTV.com' + 'name': 'myEVTV.com', }, 'nor115': { - 'name': 'NCC' + 'name': 'NCC', }, 'nor260': { - 'name': 'NDTC' + 'name': 'NDTC', }, 'nctc': { - 'name': 'Nebraska Central Telecom, Inc.' + 'name': 'Nebraska Central Telecom, Inc.', }, 'nel020': { - 'name': 'Nelsonville TV Cable' + 'name': 'Nelsonville TV Cable', }, 'nem010': { - 'name': 'Nemont' + 'name': 'Nemont', }, 'new075': { - 'name': 'New Hope Telephone Cooperative' + 'name': 'New Hope Telephone Cooperative', }, 'nor240': { - 'name': 'NICP' + 'name': 'NICP', }, 'cic010': { - 'name': 'NineStar Connect' + 'name': 'NineStar Connect', }, 'nktelco': { - 'name': 'NKTelco' + 'name': 'NKTelco', }, 'nortex': { - 'name': 'Nortex Communications' + 'name': 'Nortex Communications', }, 'nor140': { - 'name': 'North Central Telephone Cooperative' + 'name': 'North Central Telephone Cooperative', }, 'nor030': { - 'name': 'Northland Communications' + 'name': 'Northland Communications', }, 'nor075': { - 'name': 'Northwest Communications' + 'name': 'Northwest Communications', }, 'nor125': { - 'name': 'Norwood Light Broadband' + 'name': 'Norwood Light Broadband', }, 'net010': { - 'name': 'Nsight Telservices' + 'name': 'Nsight Telservices', }, 'dur010': { - 'name': 'Ntec' + 'name': 'Ntec', }, 'nts010': { - 'name': 'NTS Communications' + 'name': 'NTS Communications', }, 'new045': { - 'name': 'NU-Telecom' + 'name': 'NU-Telecom', }, 'nulink': { - 'name': 'NuLink' + 'name': 'NuLink', }, 'jam030': { - 'name': 'NVC' + 'name': 'NVC', }, 'far035': { - 'name': 'OmniTel Communications' + 'name': 'OmniTel Communications', }, 'onesource': { - 'name': 'OneSource Communications' + 'name': 'OneSource Communications', }, 'cit230': { - 'name': 'Opelika Power Services' + 'name': 'Opelika Power Services', }, 'daltonutilities': { - 'name': 'OptiLink' + 'name': 'OptiLink', }, 'mid140': { - 'name': 'OPTURA' + 'name': 'OPTURA', }, 'ote010': { - 'name': 'OTEC Communication Company' + 'name': 'OTEC Communication Company', }, 'cci020': { - 'name': 'Packerland Broadband' + 'name': 'Packerland Broadband', }, 'pan010': { - 'name': 'Panora Telco/Guthrie Center Communications' + 'name': 'Panora Telco/Guthrie Center Communications', }, 'otter': { - 'name': 'Park Region Telephone & Otter Tail Telcom' + 'name': 'Park Region Telephone & Otter Tail Telcom', }, 'mid050': { - 'name': 'Partner Communications Cooperative' + 'name': 'Partner Communications Cooperative', }, 'fib010': { - 'name': 'Pathway' + 'name': 'Pathway', }, 'paulbunyan': { - 'name': 'Paul Bunyan Communications' + 'name': 'Paul Bunyan Communications', }, 'pem020': { - 'name': 'Pembroke Telephone Company' + 'name': 'Pembroke Telephone Company', }, 'mck010': { - 'name': 'Peoples Rural Telephone Cooperative' + 'name': 'Peoples Rural Telephone Cooperative', }, 'pul010': { - 'name': 'PES Energize' + 'name': 'PES Energize', }, 'phi010': { - 'name': 'Philippi Communications System' + 'name': 'Philippi Communications System', }, 'phonoscope': { - 'name': 'Phonoscope Cable' + 'name': 'Phonoscope Cable', }, 'pin070': { - 'name': 'Pine Belt Communications, Inc.' + 'name': 'Pine Belt Communications, Inc.', }, 'weh010-pine': { - 'name': 'Pine Bluff Cable TV' + 'name': 'Pine Bluff Cable TV', }, 'pin060': { - 'name': 'Pineland Telephone Cooperative' + 'name': 'Pineland Telephone Cooperative', }, 'cam010': { - 'name': 'Pinpoint Communications' + 'name': 'Pinpoint Communications', }, 'pio060': { - 'name': 'Pioneer Broadband' + 'name': 'Pioneer Broadband', }, 'pioncomm': { - 'name': 'Pioneer Communications' + 'name': 'Pioneer Communications', }, 'pioneer': { - 'name': 'Pioneer DTV' + 'name': 'Pioneer DTV', }, 'pla020': { - 'name': 'Plant TiftNet, Inc.' + 'name': 'Plant TiftNet, Inc.', }, 'par010': { - 'name': 'PLWC' + 'name': 'PLWC', }, 'pro035': { - 'name': 'PMT' + 'name': 'PMT', }, 'vik011': { - 'name': 'Polar Cablevision' + 'name': 'Polar Cablevision', }, 'pottawatomie': { - 'name': 'Pottawatomie Telephone Co.' + 'name': 'Pottawatomie Telephone Co.', }, 'premiercomm': { - 'name': 'Premier Communications' + 'name': 'Premier Communications', }, 'psc010': { - 'name': 'PSC' + 'name': 'PSC', }, 'pan020': { - 'name': 'PTCI' + 'name': 'PTCI', }, 'qco010': { - 'name': 'QCOL' + 'name': 'QCOL', }, 'qua010': { - 'name': 'Quality Cablevision' + 'name': 'Quality Cablevision', }, 'rad010': { - 'name': 'Radcliffe Telephone Company' + 'name': 'Radcliffe Telephone Company', }, 'car040': { - 'name': 'Rainbow Communications' + 'name': 'Rainbow Communications', }, 'rai030': { - 'name': 'Rainier Connect' + 'name': 'Rainier Connect', }, 'ral010': { - 'name': 'Ralls Technologies' + 'name': 'Ralls Technologies', }, 'rct010': { - 'name': 'RC Technologies' + 'name': 'RC Technologies', }, 'red040': { - 'name': 'Red River Communications' + 'name': 'Red River Communications', }, 'ree010': { - 'name': 'Reedsburg Utility Commission' + 'name': 'Reedsburg Utility Commission', }, 'mol010': { - 'name': 'Reliance Connects- Oregon' + 'name': 'Reliance Connects- Oregon', }, 'res020': { - 'name': 'Reserve Telecommunications' + 'name': 'Reserve Telecommunications', }, 'weh010-resort': { - 'name': 'Resort TV Cable' + 'name': 'Resort TV Cable', }, 'rld010': { - 'name': 'Richland Grant Telephone Cooperative, Inc.' + 'name': 'Richland Grant Telephone Cooperative, Inc.', }, 'riv030': { - 'name': 'River Valley Telecommunications Coop' + 'name': 'River Valley Telecommunications Coop', }, 'rockportcable': { - 'name': 'Rock Port Cablevision' + 'name': 'Rock Port Cablevision', }, 'rsf010': { - 'name': 'RS Fiber' + 'name': 'RS Fiber', }, 'rtc': { - 'name': 'RTC Communication Corp' + 'name': 'RTC Communication Corp', }, 'res040': { - 'name': 'RTC-Reservation Telephone Coop.' + 'name': 'RTC-Reservation Telephone Coop.', }, 'rte010': { - 'name': 'RTEC Communications' + 'name': 'RTEC Communications', }, 'stc010': { - 'name': 'S&T' + 'name': 'S&T', }, 'san020': { - 'name': 'San Bruno Cable TV' + 'name': 'San Bruno Cable TV', }, 'san040-01': { - 'name': 'Santel' + 'name': 'Santel', }, 'sav010': { - 'name': 'SCI Broadband-Savage Communications Inc.' + 'name': 'SCI Broadband-Savage Communications Inc.', }, 'sco050': { - 'name': 'Scottsboro Electric Power Board' + 'name': 'Scottsboro Electric Power Board', }, 'scr010': { - 'name': 'Scranton Telephone Company' + 'name': 'Scranton Telephone Company', }, 'selco': { - 'name': 'SELCO' + 'name': 'SELCO', }, 'she010': { - 'name': 'Shentel' + 'name': 'Shentel', }, 'she030': { - 'name': 'Sherwood Mutual Telephone Association, Inc.' + 'name': 'Sherwood Mutual Telephone Association, Inc.', }, 'ind060-ssc': { - 'name': 'Silver Star Communications' + 'name': 'Silver Star Communications', }, 'sjoberg': { - 'name': 'Sjoberg\'s Inc.' + 'name': 'Sjoberg\'s Inc.', }, 'sou025': { - 'name': 'SKT' + 'name': 'SKT', }, 'sky050': { - 'name': 'SkyBest TV' + 'name': 'SkyBest TV', }, 'nttcsmi010': { - 'name': 'Smithville Communications' + 'name': 'Smithville Communications', }, 'woo010': { - 'name': 'Solarus' + 'name': 'Solarus', }, 'sou075': { - 'name': 'South Central Rural Telephone Cooperative' + 'name': 'South Central Rural Telephone Cooperative', }, 'sou065': { - 'name': 'South Holt Cablevision, Inc.' + 'name': 'South Holt Cablevision, Inc.', }, 'sou035': { - 'name': 'South Slope Cooperative Communications' + 'name': 'South Slope Cooperative Communications', }, 'spa020': { - 'name': 'Spanish Fork Community Network' + 'name': 'Spanish Fork Community Network', }, 'spe010': { - 'name': 'Spencer Municipal Utilities' + 'name': 'Spencer Municipal Utilities', }, 'spi005': { - 'name': 'Spillway Communications, Inc.' + 'name': 'Spillway Communications, Inc.', }, 'srt010': { - 'name': 'SRT' + 'name': 'SRT', }, 'cccsmc010': { - 'name': 'St. Maarten Cable TV' + 'name': 'St. Maarten Cable TV', }, 'sta025': { - 'name': 'Star Communications' + 'name': 'Star Communications', }, 'sco020': { - 'name': 'STE' + 'name': 'STE', }, 'uin010': { - 'name': 'STRATA Networks' + 'name': 'STRATA Networks', }, 'sum010': { - 'name': 'Sumner Cable TV' + 'name': 'Sumner Cable TV', }, 'pie010': { - 'name': 'Surry TV/PCSI TV' + 'name': 'Surry TV/PCSI TV', }, 'swa010': { - 'name': 'Swayzee Communications' + 'name': 'Swayzee Communications', }, 'sweetwater': { - 'name': 'Sweetwater Cable Television Co' + 'name': 'Sweetwater Cable Television Co', }, 'weh010-talequah': { - 'name': 'Tahlequah Cable TV' + 'name': 'Tahlequah Cable TV', }, 'tct': { - 'name': 'TCT' + 'name': 'TCT', }, 'tel050': { - 'name': 'Tele-Media Company' + 'name': 'Tele-Media Company', }, 'com050': { - 'name': 'The Community Agency' + 'name': 'The Community Agency', }, 'thr020': { - 'name': 'Three River' + 'name': 'Three River', }, 'cab140': { - 'name': 'Town & Country Technologies' + 'name': 'Town & Country Technologies', }, 'tra010': { - 'name': 'Trans-Video' + 'name': 'Trans-Video', }, 'tre010': { - 'name': 'Trenton TV Cable Company' + 'name': 'Trenton TV Cable Company', }, 'tcc': { - 'name': 'Tri County Communications Cooperative' + 'name': 'Tri County Communications Cooperative', }, 'tri025': { - 'name': 'TriCounty Telecom' + 'name': 'TriCounty Telecom', }, 'tri110': { - 'name': 'TrioTel Communications, Inc.' + 'name': 'TrioTel Communications, Inc.', }, 'tro010': { - 'name': 'Troy Cablevision, Inc.' + 'name': 'Troy Cablevision, Inc.', }, 'tsc': { - 'name': 'TSC' + 'name': 'TSC', }, 'cit220': { - 'name': 'Tullahoma Utilities Board' + 'name': 'Tullahoma Utilities Board', }, 'tvc030': { - 'name': 'TV Cable of Rensselaer' + 'name': 'TV Cable of Rensselaer', }, 'tvc015': { - 'name': 'TVC Cable' + 'name': 'TVC Cable', }, 'cab180': { - 'name': 'TVision' + 'name': 'TVision', }, 'twi040': { - 'name': 'Twin Lakes' + 'name': 'Twin Lakes', }, 'tvtinc': { - 'name': 'Twin Valley' + 'name': 'Twin Valley', }, 'uis010': { - 'name': 'Union Telephone Company' + 'name': 'Union Telephone Company', }, 'uni110': { - 'name': 'United Communications - TN' + 'name': 'United Communications - TN', }, 'uni120': { - 'name': 'United Services' + 'name': 'United Services', }, 'uss020': { - 'name': 'US Sonet' + 'name': 'US Sonet', }, 'cab060': { - 'name': 'USA Communications' + 'name': 'USA Communications', }, 'she005': { - 'name': 'USA Communications/Shellsburg, IA' + 'name': 'USA Communications/Shellsburg, IA', }, 'val040': { - 'name': 'Valley TeleCom Group' + 'name': 'Valley TeleCom Group', }, 'val025': { - 'name': 'Valley Telecommunications' + 'name': 'Valley Telecommunications', }, 'val030': { - 'name': 'Valparaiso Broadband' + 'name': 'Valparaiso Broadband', }, 'cla050': { - 'name': 'Vast Broadband' + 'name': 'Vast Broadband', }, 'sul015': { - 'name': 'Venture Communications Cooperative, Inc.' + 'name': 'Venture Communications Cooperative, Inc.', }, 'ver025': { - 'name': 'Vernon Communications Co-op' + 'name': 'Vernon Communications Co-op', }, 'weh010-vicksburg': { - 'name': 'Vicksburg Video' + 'name': 'Vicksburg Video', }, 'vis070': { - 'name': 'Vision Communications' + 'name': 'Vision Communications', }, 'volcanotel': { - 'name': 'Volcano Vision, Inc.' + 'name': 'Volcano Vision, Inc.', }, 'vol040-02': { - 'name': 'VolFirst / BLTV' + 'name': 'VolFirst / BLTV', }, 'ver070': { - 'name': 'VTel' + 'name': 'VTel', }, 'nttcvtx010': { - 'name': 'VTX1' + 'name': 'VTX1', }, 'bci010-02': { - 'name': 'Vyve Broadband' + 'name': 'Vyve Broadband', }, 'wab020': { - 'name': 'Wabash Mutual Telephone' + 'name': 'Wabash Mutual Telephone', }, 'waitsfield': { - 'name': 'Waitsfield Cable' + 'name': 'Waitsfield Cable', }, 'wal010': { - 'name': 'Walnut Communications' + 'name': 'Walnut Communications', }, 'wavebroadband': { - 'name': 'Wave' + 'name': 'Wave', }, 'wav030': { - 'name': 'Waverly Communications Utility' + 'name': 'Waverly Communications Utility', }, 'wbi010': { - 'name': 'WBI' + 'name': 'WBI', }, 'web020': { - 'name': 'Webster-Calhoun Cooperative Telephone Association' + 'name': 'Webster-Calhoun Cooperative Telephone Association', }, 'wes005': { - 'name': 'West Alabama TV Cable' + 'name': 'West Alabama TV Cable', }, 'carolinata': { - 'name': 'West Carolina Communications' + 'name': 'West Carolina Communications', }, 'wct010': { - 'name': 'West Central Telephone Association' + 'name': 'West Central Telephone Association', }, 'wes110': { - 'name': 'West River Cooperative Telephone Company' + 'name': 'West River Cooperative Telephone Company', }, 'ani030': { - 'name': 'WesTel Systems' + 'name': 'WesTel Systems', }, 'westianet': { - 'name': 'Western Iowa Networks' + 'name': 'Western Iowa Networks', }, 'nttcwhi010': { - 'name': 'Whidbey Telecom' + 'name': 'Whidbey Telecom', }, 'weh010-white': { - 'name': 'White County Cable TV' + 'name': 'White County Cable TV', }, 'wes130': { - 'name': 'Wiatel' + 'name': 'Wiatel', }, 'wik010': { - 'name': 'Wiktel' + 'name': 'Wiktel', }, 'wil070': { - 'name': 'Wilkes Communications, Inc./RiverStreet Networks' + 'name': 'Wilkes Communications, Inc./RiverStreet Networks', }, 'wil015': { - 'name': 'Wilson Communications' + 'name': 'Wilson Communications', }, 'win010': { - 'name': 'Windomnet/SMBS' + 'name': 'Windomnet/SMBS', }, 'win090': { - 'name': 'Windstream Cable TV' + 'name': 'Windstream Cable TV', }, 'wcta': { - 'name': 'Winnebago Cooperative Telecom Association' + 'name': 'Winnebago Cooperative Telecom Association', }, 'wtc010': { - 'name': 'WTC' + 'name': 'WTC', }, 'wil040': { - 'name': 'WTC Communications, Inc.' + 'name': 'WTC Communications, Inc.', }, 'wya010': { - 'name': 'Wyandotte Cable' + 'name': 'Wyandotte Cable', }, 'hin020-02': { - 'name': 'X-Stream Services' + 'name': 'X-Stream Services', }, 'xit010': { - 'name': 'XIT Communications' + 'name': 'XIT Communications', }, 'yel010': { - 'name': 'Yelcot Communications' + 'name': 'Yelcot Communications', }, 'mid180-01': { - 'name': 'yondoo' + 'name': 'yondoo', }, 'cou060': { - 'name': 'Zito Media' + 'name': 'Zito Media', }, 'slingtv': { 'name': 'Sling TV', @@ -1355,15 +1355,16 @@ MSO_INFO = { class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should end with BaseIE/InfoExtractor _SERVICE_PROVIDER_TEMPLATE = 'https://sp.auth.adobe.com/adobe-services/%s' _USER_AGENT = 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0' + _MODERN_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; rv:131.0) Gecko/20100101 Firefox/131.0' _MVPD_CACHE = 'ap-mvpd' _DOWNLOADING_LOGIN_PAGE = 'Downloading Provider Login Page' def _download_webpage_handle(self, *args, **kwargs): headers = self.geo_verification_headers() - headers.update(kwargs.get('headers', {})) + headers.update(kwargs.get('headers') or {}) kwargs['headers'] = headers - return super(AdobePassIE, self)._download_webpage_handle( + return super()._download_webpage_handle( *args, **kwargs) @staticmethod @@ -1384,7 +1385,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en def _extract_mvpd_auth(self, url, video_id, requestor_id, resource): def xml_text(xml_str, tag): return self._search_regex( - '<%s>(.+?)' % (tag, tag), xml_str, tag) + f'<{tag}>(.+?)', xml_str, tag) def is_expired(token, date_ele): token_expires = unified_timestamp(re.sub(r'[_ ]GMT', '', xml_text(token, date_ele))) @@ -1394,7 +1395,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en form_page, urlh = form_page_res post_url = self._html_search_regex(r']+action=(["\'])(?P.+?)\1', form_page, 'post url', group='url') if not re.match(r'https?://', post_url): - post_url = compat_urlparse.urljoin(urlh.url, post_url) + post_url = urllib.parse.urljoin(urlh.url, post_url) form_data = self._hidden_inputs(form_page) form_data.update(data) return self._download_webpage_handle( @@ -1414,13 +1415,13 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)' redirect_url = self._search_regex( r'(?i)]+src="(%s)' % HistoryPlayerIE._VALID_URL, + rf']+src="({HistoryPlayerIE._VALID_URL})', webpage, 'player URL') return self.url_result(player_url, HistoryPlayerIE.ie_key()) diff --git a/plugins/youtube_download/yt_dlp/extractor/aeonco.py b/plugins/youtube_download/yt_dlp/extractor/aeonco.py index 390eae32..22d0266b 100644 --- a/plugins/youtube_download/yt_dlp/extractor/aeonco.py +++ b/plugins/youtube_download/yt_dlp/extractor/aeonco.py @@ -16,8 +16,8 @@ class AeonCoIE(InfoExtractor): 'uploader': 'Semiconductor', 'uploader_id': 'semiconductor', 'uploader_url': 'https://vimeo.com/semiconductor', - 'duration': 348 - } + 'duration': 348, + }, }, { 'url': 'https://aeon.co/videos/dazzling-timelapse-shows-how-microbes-spoil-our-food-and-sometimes-enrich-it', 'md5': '03582d795382e49f2fd0b427b55de409', @@ -29,8 +29,8 @@ class AeonCoIE(InfoExtractor): 'uploader': 'Aeon Video', 'uploader_id': 'aeonvideo', 'uploader_url': 'https://vimeo.com/aeonvideo', - 'duration': 1344 - } + 'duration': 1344, + }, }, { 'url': 'https://aeon.co/videos/chew-over-the-prisoners-dilemma-and-see-if-you-can-find-the-rational-path-out', 'md5': '1cfda0bf3ae24df17d00f2c0cb6cc21b', diff --git a/plugins/youtube_download/yt_dlp/extractor/afreecatv.py b/plugins/youtube_download/yt_dlp/extractor/afreecatv.py index 3d26d9c2..aadb4d66 100644 --- a/plugins/youtube_download/yt_dlp/extractor/afreecatv.py +++ b/plugins/youtube_download/yt_dlp/extractor/afreecatv.py @@ -1,142 +1,26 @@ +import datetime as dt import functools -import re from .common import InfoExtractor +from ..networking import Request from ..utils import ( ExtractorError, OnDemandPagedList, - date_from_str, + UserNotLive, determine_ext, + filter_dict, int_or_none, - qualities, - traverse_obj, - unified_strdate, - unified_timestamp, - update_url_query, + orderedSet, + parse_iso8601, url_or_none, urlencode_postdata, - xpath_text, + urljoin, ) +from ..utils.traversal import traverse_obj -class AfreecaTVIE(InfoExtractor): - IE_NAME = 'afreecatv' - IE_DESC = 'afreecatv.com' - _VALID_URL = r'''(?x) - https?:// - (?: - (?:(?:live|afbbs|www)\.)?afreeca(?:tv)?\.com(?::\d+)? - (?: - /app/(?:index|read_ucc_bbs)\.cgi| - /player/[Pp]layer\.(?:swf|html) - )\?.*?\bnTitleNo=| - vod\.afreecatv\.com/(PLAYER/STATION|player)/ - ) - (?P\d+) - ''' +class AfreecaTVBaseIE(InfoExtractor): _NETRC_MACHINE = 'afreecatv' - _TESTS = [{ - 'url': 'http://live.afreecatv.com:8079/app/index.cgi?szType=read_ucc_bbs&szBjId=dailyapril&nStationNo=16711924&nBbsNo=18605867&nTitleNo=36164052&szSkin=', - 'md5': 'f72c89fe7ecc14c1b5ce506c4996046e', - 'info_dict': { - 'id': '36164052', - 'ext': 'mp4', - 'title': '데일리 에이프릴 요정들의 시상식!', - 'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$', - 'uploader': 'dailyapril', - 'uploader_id': 'dailyapril', - 'upload_date': '20160503', - }, - 'skip': 'Video is gone', - }, { - 'url': 'http://afbbs.afreecatv.com:8080/app/read_ucc_bbs.cgi?nStationNo=16711924&nTitleNo=36153164&szBjId=dailyapril&nBbsNo=18605867', - 'info_dict': { - 'id': '36153164', - 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'", - 'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$', - 'uploader': 'dailyapril', - 'uploader_id': 'dailyapril', - }, - 'playlist_count': 2, - 'playlist': [{ - 'md5': 'd8b7c174568da61d774ef0203159bf97', - 'info_dict': { - 'id': '36153164_1', - 'ext': 'mp4', - 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'", - 'upload_date': '20160502', - }, - }, { - 'md5': '58f2ce7f6044e34439ab2d50612ab02b', - 'info_dict': { - 'id': '36153164_2', - 'ext': 'mp4', - 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'", - 'upload_date': '20160502', - }, - }], - 'skip': 'Video is gone', - }, { - # non standard key - 'url': 'http://vod.afreecatv.com/PLAYER/STATION/20515605', - 'info_dict': { - 'id': '20170411_BE689A0E_190960999_1_2_h', - 'ext': 'mp4', - 'title': '혼자사는여자집', - 'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$', - 'uploader': '♥이슬이', - 'uploader_id': 'dasl8121', - 'upload_date': '20170411', - 'duration': 213, - }, - 'params': { - 'skip_download': True, - }, - }, { - # adult content - 'url': 'https://vod.afreecatv.com/player/97267690', - 'info_dict': { - 'id': '20180327_27901457_202289533_1', - 'ext': 'mp4', - 'title': '[생]빨개요♥ (part 1)', - 'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$', - 'uploader': '[SA]서아', - 'uploader_id': 'bjdyrksu', - 'upload_date': '20180327', - 'duration': 3601, - }, - 'params': { - 'skip_download': True, - }, - 'skip': 'The VOD does not exist', - }, { - 'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652', - 'only_matching': True, - }, { - 'url': 'https://vod.afreecatv.com/player/96753363', - 'info_dict': { - 'id': '20230108_9FF5BEE1_244432674_1', - 'ext': 'mp4', - 'uploader_id': 'rlantnghks', - 'uploader': '페이즈으', - 'duration': 10840, - 'thumbnail': 'http://videoimg.afreecatv.com/php/SnapshotLoad.php?rowKey=20230108_9FF5BEE1_244432674_1_r', - 'upload_date': '20230108', - 'title': '젠지 페이즈', - }, - 'params': { - 'skip_download': True, - }, - }] - - @staticmethod - def parse_video_key(key): - video_key = {} - m = re.match(r'^(?P\d{8})_\w+_(?P\d+)$', key) - if m: - video_key['upload_date'] = m.group('upload_date') - video_key['part'] = int(m.group('part')) - return video_key def _perform_login(self, username, password): login_form = { @@ -150,21 +34,21 @@ class AfreecaTVIE(InfoExtractor): } response = self._download_json( - 'https://login.afreecatv.com/app/LoginAction.php', None, + 'https://login.sooplive.co.kr/app/LoginAction.php', None, 'Logging in', data=urlencode_postdata(login_form)) _ERRORS = { -4: 'Your account has been suspended due to a violation of our terms and policies.', - -5: 'https://member.afreecatv.com/app/user_delete_progress.php', - -6: 'https://login.afreecatv.com/membership/changeMember.php', - -8: "Hello! AfreecaTV here.\nThe username you have entered belongs to \n an account that requires a legal guardian's consent. \nIf you wish to use our services without restriction, \nplease make sure to go through the necessary verification process.", - -9: 'https://member.afreecatv.com/app/pop_login_block.php', - -11: 'https://login.afreecatv.com/afreeca/second_login.php', - -12: 'https://member.afreecatv.com/app/user_security.php', + -5: 'https://member.sooplive.co.kr/app/user_delete_progress.php', + -6: 'https://login.sooplive.co.kr/membership/changeMember.php', + -8: "Hello! Soop here.\nThe username you have entered belongs to \n an account that requires a legal guardian's consent. \nIf you wish to use our services without restriction, \nplease make sure to go through the necessary verification process.", + -9: 'https://member.sooplive.co.kr/app/pop_login_block.php', + -11: 'https://login.sooplive.co.kr/afreeca/second_login.php', + -12: 'https://member.sooplive.co.kr/app/user_security.php', 0: 'The username does not exist or you have entered the wrong password.', -1: 'The username does not exist or you have entered the wrong password.', -3: 'You have entered your username/password incorrectly.', - -7: 'You cannot use your Global AfreecaTV account to access Korean AfreecaTV.', + -7: 'You cannot use your Global Soop account to access Korean Soop.', -10: 'Sorry for the inconvenience. \nYour account has been blocked due to an unauthorized access. \nPlease contact our Help Center for assistance.', -32008: 'You have failed to log in. Please contact our Help Center.', } @@ -173,169 +57,206 @@ class AfreecaTVIE(InfoExtractor): if result != 1: error = _ERRORS.get(result, 'You have failed to log in.') raise ExtractorError( - 'Unable to login: %s said: %s' % (self.IE_NAME, error), + f'Unable to login: {self.IE_NAME} said: {error}', expected=True) + def _call_api(self, endpoint, display_id, data=None, headers=None, query=None): + return self._download_json(Request( + f'https://api.m.sooplive.co.kr/{endpoint}', + data=data, headers=headers, query=query, + extensions={'legacy_ssl': True}), display_id, + 'Downloading API JSON', 'Unable to download API JSON') + + @staticmethod + def _fixup_thumb(thumb_url): + if not url_or_none(thumb_url): + return None + # Core would determine_ext as 'php' from the url, so we need to provide the real ext + # See: https://github.com/yt-dlp/yt-dlp/issues/11537 + return [{'url': thumb_url, 'ext': 'jpg'}] + + +class AfreecaTVIE(AfreecaTVBaseIE): + IE_NAME = 'soop' + IE_DESC = 'sooplive.co.kr' + _VALID_URL = r'https?://vod\.(?:sooplive\.co\.kr|afreecatv\.com)/(?:PLAYER/STATION|player)/(?P\d+)/?(?:$|[?#&])' + _TESTS = [{ + 'url': 'https://vod.sooplive.co.kr/player/96753363', + 'info_dict': { + 'id': '20230108_9FF5BEE1_244432674_1', + 'ext': 'mp4', + 'uploader_id': 'rlantnghks', + 'uploader': '페이즈으', + 'duration': 10840, + 'thumbnail': r're:https?://videoimg\.(?:sooplive\.co\.kr|afreecatv\.com)/.+', + 'upload_date': '20230108', + 'timestamp': 1673186405, + 'title': '젠지 페이즈', + }, + 'params': { + 'skip_download': True, + }, + }, { + # non standard key + 'url': 'http://vod.sooplive.co.kr/PLAYER/STATION/20515605', + 'info_dict': { + 'id': '20170411_BE689A0E_190960999_1_2_h', + 'ext': 'mp4', + 'title': '혼자사는여자집', + 'thumbnail': r're:https?://(?:video|st)img\.(?:sooplive\.co\.kr|afreecatv\.com)/.+', + 'uploader': '♥이슬이', + 'uploader_id': 'dasl8121', + 'upload_date': '20170411', + 'timestamp': 1491929865, + 'duration': 213, + }, + 'params': { + 'skip_download': True, + }, + }, { + # adult content + 'url': 'https://vod.sooplive.co.kr/player/97267690', + 'info_dict': { + 'id': '20180327_27901457_202289533_1', + 'ext': 'mp4', + 'title': '[생]빨개요♥ (part 1)', + 'thumbnail': r're:https?://(?:video|st)img\.(?:sooplive\.co\.kr|afreecatv\.com)/.+', + 'uploader': '[SA]서아', + 'uploader_id': 'bjdyrksu', + 'upload_date': '20180327', + 'duration': 3601, + }, + 'params': { + 'skip_download': True, + }, + 'skip': 'The VOD does not exist', + }, { + # adult content + 'url': 'https://vod.sooplive.co.kr/player/70395877', + 'only_matching': True, + }, { + # subscribers only + 'url': 'https://vod.sooplive.co.kr/player/104647403', + 'only_matching': True, + }, { + # private + 'url': 'https://vod.sooplive.co.kr/player/81669846', + 'only_matching': True, + }] + def _real_extract(self, url): video_id = self._match_id(url) - - partial_view = False - adult_view = False - for _ in range(2): - data = self._download_json( - 'https://api.m.afreecatv.com/station/video/a/view', - video_id, headers={'Referer': url}, data=urlencode_postdata({ - 'nTitleNo': video_id, - 'nApiLevel': 10, - }))['data'] - if traverse_obj(data, ('code', {int})) == -6221: - raise ExtractorError('The VOD does not exist', expected=True) - query = { + data = self._call_api( + 'station/video/a/view', video_id, headers={'Referer': url}, + data=urlencode_postdata({ 'nTitleNo': video_id, - 'nStationNo': data['station_no'], - 'nBbsNo': data['bbs_no'], - } - if partial_view: - query['partialView'] = 'SKIP_ADULT' - if adult_view: - query['adultView'] = 'ADULT_VIEW' - video_xml = self._download_xml( - 'http://afbbs.afreecatv.com:8080/api/video/get_video_info.php', - video_id, 'Downloading video info XML%s' - % (' (skipping adult)' if partial_view else ''), - video_id, headers={ - 'Referer': url, - }, query=query) + 'nApiLevel': 10, + }))['data'] - flag = xpath_text(video_xml, './track/flag', 'flag', default=None) - if flag and flag == 'SUCCEED': - break - if flag == 'PARTIAL_ADULT': - self.report_warning( - 'In accordance with local laws and regulations, underage users are restricted from watching adult content. ' - 'Only content suitable for all ages will be downloaded. ' - 'Provide account credentials if you wish to download restricted content.') - partial_view = True - continue - elif flag == 'ADULT': - if not adult_view: - adult_view = True - continue - error = 'Only users older than 19 are able to watch this video. Provide account credentials to download this content.' - else: - error = flag - raise ExtractorError( - '%s said: %s' % (self.IE_NAME, error), expected=True) - else: - raise ExtractorError('Unable to download video info') + error_code = traverse_obj(data, ('code', {int})) + if error_code == -6221: + raise ExtractorError('The VOD does not exist', expected=True) + elif error_code == -6205: + raise ExtractorError('This VOD is private', expected=True) - video_element = video_xml.findall('./track/video')[-1] - if video_element is None or video_element.text is None: - raise ExtractorError( - 'Video %s does not exist' % video_id, expected=True) - - video_url = video_element.text.strip() - - title = xpath_text(video_xml, './track/title', 'title', fatal=True) - - uploader = xpath_text(video_xml, './track/nickname', 'uploader') - uploader_id = xpath_text(video_xml, './track/bj_id', 'uploader id') - duration = int_or_none(xpath_text( - video_xml, './track/duration', 'duration')) - thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail') - - common_entry = { - 'uploader': uploader, - 'uploader_id': uploader_id, - 'thumbnail': thumbnail, - } - - info = common_entry.copy() - info.update({ - 'id': video_id, - 'title': title, - 'duration': duration, + common_info = traverse_obj(data, { + 'title': ('title', {str}), + 'uploader': ('writer_nick', {str}), + 'uploader_id': ('bj_id', {str}), + 'duration': ('total_file_duration', {int_or_none(scale=1000)}), + 'thumbnails': ('thumb', {self._fixup_thumb}), }) - if not video_url: - entries = [] - file_elements = video_element.findall('./file') - one = len(file_elements) == 1 - for file_num, file_element in enumerate(file_elements, start=1): - file_url = url_or_none(file_element.text) - if not file_url: - continue - key = file_element.get('key', '') - upload_date = unified_strdate(self._search_regex( - r'^(\d{8})_', key, 'upload date', default=None)) - if upload_date is not None: - # sometimes the upload date isn't included in the file name - # instead, another random ID is, which may parse as a valid - # date but be wildly out of a reasonable range - parsed_date = date_from_str(upload_date) - if parsed_date.year < 2000 or parsed_date.year >= 2100: - upload_date = None - file_duration = int_or_none(file_element.get('duration')) - format_id = key if key else '%s_%s' % (video_id, file_num) - if determine_ext(file_url) == 'm3u8': - formats = self._extract_m3u8_formats( - file_url, video_id, 'mp4', entry_protocol='m3u8_native', - m3u8_id='hls', - note='Downloading part %d m3u8 information' % file_num) - else: - formats = [{ - 'url': file_url, - 'format_id': 'http', - }] - if not formats and not self.get_param('ignore_no_formats'): - continue - file_info = common_entry.copy() - file_info.update({ - 'id': format_id, - 'title': title if one else '%s (part %d)' % (title, file_num), - 'upload_date': upload_date, - 'duration': file_duration, - 'formats': formats, - }) - entries.append(file_info) - entries_info = info.copy() - entries_info.update({ - '_type': 'multi_video', - 'entries': entries, - }) - return entries_info + entries = [] + for file_num, file_element in enumerate( + traverse_obj(data, ('files', lambda _, v: url_or_none(v['file']))), start=1): + file_url = file_element['file'] + if determine_ext(file_url) == 'm3u8': + formats = self._extract_m3u8_formats( + file_url, video_id, 'mp4', m3u8_id='hls', + note=f'Downloading part {file_num} m3u8 information') + else: + formats = [{ + 'url': file_url, + 'format_id': 'http', + }] - info = { - 'id': video_id, - 'title': title, - 'uploader': uploader, - 'uploader_id': uploader_id, - 'duration': duration, - 'thumbnail': thumbnail, - } - - if determine_ext(video_url) == 'm3u8': - info['formats'] = self._extract_m3u8_formats( - video_url, video_id, 'mp4', entry_protocol='m3u8_native', - m3u8_id='hls') - else: - app, playpath = video_url.split('mp4:') - info.update({ - 'url': app, - 'ext': 'flv', - 'play_path': 'mp4:' + playpath, - 'rtmp_live': True, # downloading won't end without this + entries.append({ + **common_info, + 'id': file_element.get('file_info_key') or f'{video_id}_{file_num}', + 'title': f'{common_info.get("title") or "Untitled"} (part {file_num})', + 'formats': formats, + **traverse_obj(file_element, { + 'duration': ('duration', {int_or_none(scale=1000)}), + 'timestamp': ('file_start', {parse_iso8601(delimiter=' ', timezone=dt.timedelta(hours=9))}), + }), }) - return info + if traverse_obj(data, ('adult_status', {str})) == 'notLogin': + if not entries: + self.raise_login_required( + 'Only users older than 19 are able to watch this video', method='password') + self.report_warning( + 'In accordance with local laws and regulations, underage users are ' + 'restricted from watching adult content. Only content suitable for all ' + f'ages will be downloaded. {self._login_hint("password")}') + + if not entries and traverse_obj(data, ('sub_upload_type', {str})): + self.raise_login_required('This VOD is for subscribers only', method='password') + + if len(entries) == 1: + return { + **entries[0], + 'title': common_info.get('title'), + } + + common_info['timestamp'] = traverse_obj(entries, (..., 'timestamp'), get_all=False) + + return self.playlist_result(entries, video_id, multi_video=True, **common_info) -class AfreecaTVLiveIE(AfreecaTVIE): # XXX: Do not subclass from concrete IE - - IE_NAME = 'afreecatv:live' - _VALID_URL = r'https?://play\.afreeca(?:tv)?\.com/(?P[^/]+)(?:/(?P\d+))?' +class AfreecaTVCatchStoryIE(AfreecaTVBaseIE): + IE_NAME = 'soop:catchstory' + IE_DESC = 'sooplive.co.kr catch story' + _VALID_URL = r'https?://vod\.(?:sooplive\.co\.kr|afreecatv\.com)/player/(?P\d+)/catchstory' _TESTS = [{ - 'url': 'https://play.afreecatv.com/pyh3646/237852185', + 'url': 'https://vod.sooplive.co.kr/player/103247/catchstory', + 'info_dict': { + 'id': '103247', + }, + 'playlist_count': 2, + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + data = self._call_api( + 'catchstory/a/view', video_id, headers={'Referer': url}, + query={'aStoryListIdx': '', 'nStoryIdx': video_id}) + + return self.playlist_result(self._entries(data), video_id) + + def _entries(self, data): + # 'files' is always a list with 1 element + yield from traverse_obj(data, ( + 'data', lambda _, v: v['story_type'] == 'catch', + 'catch_list', lambda _, v: v['files'][0]['file'], { + 'id': ('files', 0, 'file_info_key', {str}), + 'url': ('files', 0, 'file', {url_or_none}), + 'duration': ('files', 0, 'duration', {int_or_none(scale=1000)}), + 'title': ('title', {str}), + 'uploader': ('writer_nick', {str}), + 'uploader_id': ('writer_id', {str}), + 'thumbnails': ('thumb', {self._fixup_thumb}), + 'timestamp': ('write_timestamp', {int_or_none}), + })) + + +class AfreecaTVLiveIE(AfreecaTVBaseIE): + IE_NAME = 'soop:live' + IE_DESC = 'sooplive.co.kr livestreams' + _VALID_URL = r'https?://play\.(?:sooplive\.co\.kr|afreecatv\.com)/(?P[^/?#]+)(?:/(?P\d+))?' + _TESTS = [{ + 'url': 'https://play.sooplive.co.kr/pyh3646/237852185', 'info_dict': { 'id': '237852185', 'ext': 'mp4', @@ -347,94 +268,121 @@ class AfreecaTVLiveIE(AfreecaTVIE): # XXX: Do not subclass from concrete IE }, 'skip': 'Livestream has ended', }, { - 'url': 'http://play.afreeca.com/pyh3646/237852185', + 'url': 'https://play.sooplive.co.kr/pyh3646/237852185', 'only_matching': True, }, { - 'url': 'http://play.afreeca.com/pyh3646', + 'url': 'https://play.sooplive.co.kr/pyh3646', 'only_matching': True, }] - _LIVE_API_URL = 'https://live.afreecatv.com/afreeca/player_live_api.php' + _LIVE_API_URL = 'https://live.sooplive.co.kr/afreeca/player_live_api.php' + _WORKING_CDNS = [ + 'gcp_cdn', # live-global-cdn-v02.sooplive.co.kr + 'gs_cdn_pc_app', # pc-app.stream.sooplive.co.kr + 'gs_cdn_mobile_web', # mobile-web.stream.sooplive.co.kr + 'gs_cdn_pc_web', # pc-web.stream.sooplive.co.kr + ] + _BAD_CDNS = [ + 'gs_cdn', # chromecast.afreeca.gscdn.com (cannot resolve) + 'gs_cdn_chromecast', # chromecast.stream.sooplive.co.kr (HTTP Error 400) + 'azure_cdn', # live-global-cdn-v01.sooplive.co.kr (cannot resolve) + 'aws_cf', # live-global-cdn-v03.sooplive.co.kr (cannot resolve) + 'kt_cdn', # kt.stream.sooplive.co.kr (HTTP Error 400) + ] - _QUALITIES = ('sd', 'hd', 'hd2k', 'original') + def _extract_formats(self, channel_info, broadcast_no, aid): + stream_base_url = channel_info.get('RMD') or 'https://livestream-manager.sooplive.co.kr' + + # If user has not passed CDN IDs, try API-provided CDN ID followed by other working CDN IDs + default_cdn_ids = orderedSet([ + *traverse_obj(channel_info, ('CDN', {str}, all, lambda _, v: v not in self._BAD_CDNS)), + *self._WORKING_CDNS, + ]) + cdn_ids = self._configuration_arg('cdn', default_cdn_ids) + + for attempt, cdn_id in enumerate(cdn_ids, start=1): + m3u8_url = traverse_obj(self._download_json( + urljoin(stream_base_url, 'broad_stream_assign.html'), broadcast_no, + f'Downloading {cdn_id} stream info', f'Unable to download {cdn_id} stream info', + fatal=False, query={ + 'return_type': cdn_id, + 'broad_key': f'{broadcast_no}-common-master-hls', + }), ('view_url', {url_or_none})) + try: + return self._extract_m3u8_formats( + m3u8_url, broadcast_no, 'mp4', m3u8_id='hls', query={'aid': aid}, + headers={'Referer': 'https://play.sooplive.co.kr/'}) + except ExtractorError as e: + if attempt == len(cdn_ids): + raise + self.report_warning( + f'{e.cause or e.msg}. Retrying... (attempt {attempt} of {len(cdn_ids)})') def _real_extract(self, url): broadcaster_id, broadcast_no = self._match_valid_url(url).group('id', 'bno') - password = self.get_param('videopassword') + channel_info = traverse_obj(self._download_json( + self._LIVE_API_URL, broadcaster_id, data=urlencode_postdata({'bid': broadcaster_id})), + ('CHANNEL', {dict})) or {} - info = self._download_json(self._LIVE_API_URL, broadcaster_id, fatal=False, - data=urlencode_postdata({'bid': broadcaster_id})) or {} - channel_info = info.get('CHANNEL') or {} broadcaster_id = channel_info.get('BJID') or broadcaster_id broadcast_no = channel_info.get('BNO') or broadcast_no - password_protected = channel_info.get('BPWD') if not broadcast_no: - raise ExtractorError(f'Unable to extract broadcast number ({broadcaster_id} may not be live)', expected=True) - if password_protected == 'Y' and password is None: + result = channel_info.get('RESULT') + if result == 0: + raise UserNotLive(video_id=broadcaster_id) + elif result == -6: + self.raise_login_required( + 'This channel is streaming for subscribers only', method='password') + raise ExtractorError('Unable to extract broadcast number') + + password = self.get_param('videopassword') + if channel_info.get('BPWD') == 'Y' and password is None: raise ExtractorError( 'This livestream is protected by a password, use the --video-password option', expected=True) - formats = [] - quality_key = qualities(self._QUALITIES) - for quality_str in self._QUALITIES: - params = { + token_info = traverse_obj(self._download_json( + self._LIVE_API_URL, broadcast_no, 'Downloading access token for stream', + 'Unable to download access token for stream', data=urlencode_postdata(filter_dict({ 'bno': broadcast_no, 'stream_type': 'common', 'type': 'aid', - 'quality': quality_str, - } - if password is not None: - params['pwd'] = password - aid_response = self._download_json( - self._LIVE_API_URL, broadcast_no, fatal=False, - data=urlencode_postdata(params), - note=f'Downloading access token for {quality_str} stream', - errnote=f'Unable to download access token for {quality_str} stream') - aid = traverse_obj(aid_response, ('CHANNEL', 'AID')) - if not aid: - continue + 'quality': 'master', + 'pwd': password, + }))), ('CHANNEL', {dict})) or {} + aid = token_info.get('AID') + if not aid: + result = token_info.get('RESULT') + if result == 0: + raise ExtractorError('This livestream has ended', expected=True) + elif result == -6: + self.raise_login_required('This livestream is for subscribers only', method='password') + raise ExtractorError('Unable to extract access token') - stream_base_url = channel_info.get('RMD') or 'https://livestream-manager.afreecatv.com' - stream_info = self._download_json( - f'{stream_base_url}/broad_stream_assign.html', broadcast_no, fatal=False, - query={ - 'return_type': channel_info.get('CDN', 'gcp_cdn'), - 'broad_key': f'{broadcast_no}-common-{quality_str}-hls', - }, - note=f'Downloading metadata for {quality_str} stream', - errnote=f'Unable to download metadata for {quality_str} stream') or {} + formats = self._extract_formats(channel_info, broadcast_no, aid) - if stream_info.get('view_url'): - formats.append({ - 'format_id': quality_str, - 'url': update_url_query(stream_info['view_url'], {'aid': aid}), - 'ext': 'mp4', - 'protocol': 'm3u8', - 'quality': quality_key(quality_str), - }) - - station_info = self._download_json( - 'https://st.afreecatv.com/api/get_station_status.php', broadcast_no, - query={'szBjId': broadcaster_id}, fatal=False, - note='Downloading channel metadata', errnote='Unable to download channel metadata') or {} + station_info = traverse_obj(self._download_json( + 'https://st.sooplive.co.kr/api/get_station_status.php', broadcast_no, + 'Downloading channel metadata', 'Unable to download channel metadata', + query={'szBjId': broadcaster_id}, fatal=False), {dict}) or {} return { 'id': broadcast_no, 'title': channel_info.get('TITLE') or station_info.get('station_title'), 'uploader': channel_info.get('BJNICK') or station_info.get('station_name'), 'uploader_id': broadcaster_id, - 'timestamp': unified_timestamp(station_info.get('broad_start')), + 'timestamp': parse_iso8601(station_info.get('broad_start'), delimiter=' ', timezone=dt.timedelta(hours=9)), 'formats': formats, 'is_live': True, + 'http_headers': {'Referer': url}, } -class AfreecaTVUserIE(InfoExtractor): - IE_NAME = 'afreecatv:user' - _VALID_URL = r'https?://bj\.afreeca(?:tv)?\.com/(?P[^/]+)/vods/?(?P[^/]+)?' +class AfreecaTVUserIE(AfreecaTVBaseIE): + IE_NAME = 'soop:user' + _VALID_URL = r'https?://ch\.(?:sooplive\.co\.kr|afreecatv\.com)/(?P[^/?#]+)/vods/?(?P[^/?#]+)?' _TESTS = [{ - 'url': 'https://bj.afreecatv.com/ryuryu24/vods/review', + 'url': 'https://ch.sooplive.co.kr/ryuryu24/vods/review', 'info_dict': { '_type': 'playlist', 'id': 'ryuryu24', @@ -442,7 +390,7 @@ class AfreecaTVUserIE(InfoExtractor): }, 'playlist_count': 218, }, { - 'url': 'https://bj.afreecatv.com/parang1995/vods/highlight', + 'url': 'https://ch.sooplive.co.kr/parang1995/vods/highlight', 'info_dict': { '_type': 'playlist', 'id': 'parang1995', @@ -450,7 +398,7 @@ class AfreecaTVUserIE(InfoExtractor): }, 'playlist_count': 997, }, { - 'url': 'https://bj.afreecatv.com/ryuryu24/vods', + 'url': 'https://ch.sooplive.co.kr/ryuryu24/vods', 'info_dict': { '_type': 'playlist', 'id': 'ryuryu24', @@ -458,7 +406,7 @@ class AfreecaTVUserIE(InfoExtractor): }, 'playlist_count': 221, }, { - 'url': 'https://bj.afreecatv.com/ryuryu24/vods/balloonclip', + 'url': 'https://ch.sooplive.co.kr/ryuryu24/vods/balloonclip', 'info_dict': { '_type': 'playlist', 'id': 'ryuryu24', @@ -470,12 +418,12 @@ class AfreecaTVUserIE(InfoExtractor): def _fetch_page(self, user_id, user_type, page): page += 1 - info = self._download_json(f'https://bjapi.afreecatv.com/api/{user_id}/vods/{user_type}', user_id, + info = self._download_json(f'https://chapi.sooplive.co.kr/api/{user_id}/vods/{user_type}', user_id, query={'page': page, 'per_page': self._PER_PAGE, 'orderby': 'reg_date'}, note=f'Downloading {user_type} video page {page}') for item in info['data']: yield self.url_result( - f'https://vod.afreecatv.com/player/{item["title_no"]}/', AfreecaTVIE, item['title_no']) + f'https://vod.sooplive.co.kr/player/{item["title_no"]}/', AfreecaTVIE, item['title_no']) def _real_extract(self, url): user_id, user_type = self._match_valid_url(url).group('id', 'slug_type') diff --git a/plugins/youtube_download/yt_dlp/extractor/agora.py b/plugins/youtube_download/yt_dlp/extractor/agora.py index abb2d3ff..e040db60 100644 --- a/plugins/youtube_download/yt_dlp/extractor/agora.py +++ b/plugins/youtube_download/yt_dlp/extractor/agora.py @@ -146,7 +146,7 @@ class TokFMPodcastIE(InfoExtractor): 'url': 'https://audycje.tokfm.pl/podcast/91275,-Systemowy-rasizm-Czy-zamieszki-w-USA-po-morderstwie-w-Minneapolis-doprowadza-do-zmian-w-sluzbach-panstwowych', 'info_dict': { 'id': '91275', - 'ext': 'aac', + 'ext': 'mp3', 'title': 'md5:a9b15488009065556900169fb8061cce', 'episode': 'md5:a9b15488009065556900169fb8061cce', 'series': 'Analizy', @@ -164,23 +164,20 @@ class TokFMPodcastIE(InfoExtractor): raise ExtractorError('No such podcast', expected=True) metadata = metadata[0] - formats = [] - for ext in ('aac', 'mp3'): - url_data = self._download_json( - f'https://api.podcast.radioagora.pl/api4/getSongUrl?podcast_id={media_id}&device_id={uuid.uuid4()}&ppre=false&audio={ext}', - media_id, 'Downloading podcast %s URL' % ext) - # prevents inserting the mp3 (default) multiple times - if 'link_ssl' in url_data and f'.{ext}' in url_data['link_ssl']: - formats.append({ - 'url': url_data['link_ssl'], - 'ext': ext, - 'vcodec': 'none', - 'acodec': ext, - }) + mp3_url = self._download_json( + 'https://api.podcast.radioagora.pl/api4/getSongUrl', + media_id, 'Downloading podcast mp3 URL', query={ + 'podcast_id': media_id, + 'device_id': str(uuid.uuid4()), + 'ppre': 'false', + 'audio': 'mp3', + })['link_ssl'] return { 'id': media_id, - 'formats': formats, + 'url': mp3_url, + 'vcodec': 'none', + 'ext': 'mp3', 'title': metadata.get('podcast_name'), 'series': metadata.get('series_name'), 'episode': metadata.get('podcast_name'), @@ -206,8 +203,8 @@ class TokFMAuditionIE(InfoExtractor): } @staticmethod - def _create_url(id): - return f'https://audycje.tokfm.pl/audycja/{id}' + def _create_url(video_id): + return f'https://audycje.tokfm.pl/audycja/{video_id}' def _real_extract(self, url): audition_id = self._match_id(url) diff --git a/plugins/youtube_download/yt_dlp/extractor/airmozilla.py b/plugins/youtube_download/yt_dlp/extractor/airmozilla.py deleted file mode 100644 index 669556b9..00000000 --- a/plugins/youtube_download/yt_dlp/extractor/airmozilla.py +++ /dev/null @@ -1,63 +0,0 @@ -import re - -from .common import InfoExtractor -from ..utils import ( - int_or_none, - parse_duration, - parse_iso8601, -) - - -class AirMozillaIE(InfoExtractor): - _VALID_URL = r'https?://air\.mozilla\.org/(?P[0-9a-z-]+)/?' - _TEST = { - 'url': 'https://air.mozilla.org/privacy-lab-a-meetup-for-privacy-minded-people-in-san-francisco/', - 'md5': '8d02f53ee39cf006009180e21df1f3ba', - 'info_dict': { - 'id': '6x4q2w', - 'ext': 'mp4', - 'title': 'Privacy Lab - a meetup for privacy minded people in San Francisco', - 'thumbnail': r're:https?://.*/poster\.jpg', - 'description': 'Brings together privacy professionals and others interested in privacy at for-profits, non-profits, and NGOs in an effort to contribute to the state of the ecosystem...', - 'timestamp': 1422487800, - 'upload_date': '20150128', - 'location': 'SFO Commons', - 'duration': 3780, - 'view_count': int, - 'categories': ['Main', 'Privacy'], - } - } - - def _real_extract(self, url): - display_id = self._match_id(url) - webpage = self._download_webpage(url, display_id) - video_id = self._html_search_regex(r'//vid\.ly/(.*?)/embed', webpage, 'id') - - embed_script = self._download_webpage('https://vid.ly/{0}/embed'.format(video_id), video_id) - jwconfig = self._parse_json(self._search_regex( - r'initCallback\((.*)\);', embed_script, 'metadata'), video_id)['config'] - - info_dict = self._parse_jwplayer_data(jwconfig, video_id) - view_count = int_or_none(self._html_search_regex( - r'Views since archived: ([0-9]+)', - webpage, 'view count', fatal=False)) - timestamp = parse_iso8601(self._html_search_regex( - r'