_utils.py 182 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606
  1. import base64
  2. import binascii
  3. import calendar
  4. import codecs
  5. import collections
  6. import collections.abc
  7. import contextlib
  8. import datetime as dt
  9. import email.header
  10. import email.utils
  11. import errno
  12. import hashlib
  13. import hmac
  14. import html.entities
  15. import html.parser
  16. import inspect
  17. import io
  18. import itertools
  19. import json
  20. import locale
  21. import math
  22. import mimetypes
  23. import netrc
  24. import operator
  25. import os
  26. import platform
  27. import random
  28. import re
  29. import shlex
  30. import socket
  31. import ssl
  32. import struct
  33. import subprocess
  34. import sys
  35. import tempfile
  36. import time
  37. import traceback
  38. import types
  39. import unicodedata
  40. import urllib.error
  41. import urllib.parse
  42. import urllib.request
  43. import xml.etree.ElementTree
  44. from . import traversal
  45. from ..compat import functools # isort: split
  46. from ..compat import (
  47. compat_etree_fromstring,
  48. compat_expanduser,
  49. compat_HTMLParseError,
  50. compat_os_name,
  51. )
  52. from ..dependencies import xattr
  53. __name__ = __name__.rsplit('.', 1)[0] # noqa: A001: Pretend to be the parent module
  54. # This is not clearly defined otherwise
  55. compiled_regex_type = type(re.compile(''))
  56. class NO_DEFAULT:
  57. pass
  58. def IDENTITY(x):
  59. return x
  60. ENGLISH_MONTH_NAMES = [
  61. 'January', 'February', 'March', 'April', 'May', 'June',
  62. 'July', 'August', 'September', 'October', 'November', 'December']
  63. MONTH_NAMES = {
  64. 'en': ENGLISH_MONTH_NAMES,
  65. 'fr': [
  66. 'janvier', 'février', 'mars', 'avril', 'mai', 'juin',
  67. 'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'],
  68. # these follow the genitive grammatical case (dopełniacz)
  69. # some websites might be using nominative, which will require another month list
  70. # https://en.wikibooks.org/wiki/Polish/Noun_cases
  71. 'pl': ['stycznia', 'lutego', 'marca', 'kwietnia', 'maja', 'czerwca',
  72. 'lipca', 'sierpnia', 'września', 'października', 'listopada', 'grudnia'],
  73. }
  74. # From https://github.com/python/cpython/blob/3.11/Lib/email/_parseaddr.py#L36-L42
  75. TIMEZONE_NAMES = {
  76. 'UT': 0, 'UTC': 0, 'GMT': 0, 'Z': 0,
  77. 'AST': -4, 'ADT': -3, # Atlantic (used in Canada)
  78. 'EST': -5, 'EDT': -4, # Eastern
  79. 'CST': -6, 'CDT': -5, # Central
  80. 'MST': -7, 'MDT': -6, # Mountain
  81. 'PST': -8, 'PDT': -7, # Pacific
  82. }
  83. # needed for sanitizing filenames in restricted mode
  84. ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',
  85. itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUY', ['TH', 'ss'],
  86. 'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuy', ['th'], 'y')))
  87. DATE_FORMATS = (
  88. '%d %B %Y',
  89. '%d %b %Y',
  90. '%B %d %Y',
  91. '%B %dst %Y',
  92. '%B %dnd %Y',
  93. '%B %drd %Y',
  94. '%B %dth %Y',
  95. '%b %d %Y',
  96. '%b %dst %Y',
  97. '%b %dnd %Y',
  98. '%b %drd %Y',
  99. '%b %dth %Y',
  100. '%b %dst %Y %I:%M',
  101. '%b %dnd %Y %I:%M',
  102. '%b %drd %Y %I:%M',
  103. '%b %dth %Y %I:%M',
  104. '%Y %m %d',
  105. '%Y-%m-%d',
  106. '%Y.%m.%d.',
  107. '%Y/%m/%d',
  108. '%Y/%m/%d %H:%M',
  109. '%Y/%m/%d %H:%M:%S',
  110. '%Y%m%d%H%M',
  111. '%Y%m%d%H%M%S',
  112. '%Y%m%d',
  113. '%Y-%m-%d %H:%M',
  114. '%Y-%m-%d %H:%M:%S',
  115. '%Y-%m-%d %H:%M:%S.%f',
  116. '%Y-%m-%d %H:%M:%S:%f',
  117. '%d.%m.%Y %H:%M',
  118. '%d.%m.%Y %H.%M',
  119. '%Y-%m-%dT%H:%M:%SZ',
  120. '%Y-%m-%dT%H:%M:%S.%fZ',
  121. '%Y-%m-%dT%H:%M:%S.%f0Z',
  122. '%Y-%m-%dT%H:%M:%S',
  123. '%Y-%m-%dT%H:%M:%S.%f',
  124. '%Y-%m-%dT%H:%M',
  125. '%b %d %Y at %H:%M',
  126. '%b %d %Y at %H:%M:%S',
  127. '%B %d %Y at %H:%M',
  128. '%B %d %Y at %H:%M:%S',
  129. '%H:%M %d-%b-%Y',
  130. )
  131. DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
  132. DATE_FORMATS_DAY_FIRST.extend([
  133. '%d-%m-%Y',
  134. '%d.%m.%Y',
  135. '%d.%m.%y',
  136. '%d/%m/%Y',
  137. '%d/%m/%y',
  138. '%d/%m/%Y %H:%M:%S',
  139. '%d-%m-%Y %H:%M',
  140. '%H:%M %d/%m/%Y',
  141. ])
  142. DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
  143. DATE_FORMATS_MONTH_FIRST.extend([
  144. '%m-%d-%Y',
  145. '%m.%d.%Y',
  146. '%m/%d/%Y',
  147. '%m/%d/%y',
  148. '%m/%d/%Y %H:%M:%S',
  149. ])
  150. PACKED_CODES_RE = r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)"
  151. JSON_LD_RE = r'(?is)<script[^>]+type=(["\']?)application/ld\+json\1[^>]*>\s*(?P<json_ld>{.+?}|\[.+?\])\s*</script>'
  152. NUMBER_RE = r'\d+(?:\.\d+)?'
  153. @functools.cache
  154. def preferredencoding():
  155. """Get preferred encoding.
  156. Returns the best encoding scheme for the system, based on
  157. locale.getpreferredencoding() and some further tweaks.
  158. """
  159. try:
  160. pref = locale.getpreferredencoding()
  161. 'TEST'.encode(pref)
  162. except Exception:
  163. pref = 'UTF-8'
  164. return pref
  165. def write_json_file(obj, fn):
  166. """ Encode obj as JSON and write it to fn, atomically if possible """
  167. tf = tempfile.NamedTemporaryFile(
  168. prefix=f'{os.path.basename(fn)}.', dir=os.path.dirname(fn),
  169. suffix='.tmp', delete=False, mode='w', encoding='utf-8')
  170. try:
  171. with tf:
  172. json.dump(obj, tf, ensure_ascii=False)
  173. if sys.platform == 'win32':
  174. # Need to remove existing file on Windows, else os.rename raises
  175. # WindowsError or FileExistsError.
  176. with contextlib.suppress(OSError):
  177. os.unlink(fn)
  178. with contextlib.suppress(OSError):
  179. mask = os.umask(0)
  180. os.umask(mask)
  181. os.chmod(tf.name, 0o666 & ~mask)
  182. os.rename(tf.name, fn)
  183. except Exception:
  184. with contextlib.suppress(OSError):
  185. os.remove(tf.name)
  186. raise
  187. def find_xpath_attr(node, xpath, key, val=None):
  188. """ Find the xpath xpath[@key=val] """
  189. assert re.match(r'^[a-zA-Z_-]+$', key)
  190. expr = xpath + (f'[@{key}]' if val is None else f"[@{key}='{val}']")
  191. return node.find(expr)
  192. # On python2.6 the xml.etree.ElementTree.Element methods don't support
  193. # the namespace parameter
  194. def xpath_with_ns(path, ns_map):
  195. components = [c.split(':') for c in path.split('/')]
  196. replaced = []
  197. for c in components:
  198. if len(c) == 1:
  199. replaced.append(c[0])
  200. else:
  201. ns, tag = c
  202. replaced.append(f'{{{ns_map[ns]}}}{tag}')
  203. return '/'.join(replaced)
  204. def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
  205. def _find_xpath(xpath):
  206. return node.find(xpath)
  207. if isinstance(xpath, str):
  208. n = _find_xpath(xpath)
  209. else:
  210. for xp in xpath:
  211. n = _find_xpath(xp)
  212. if n is not None:
  213. break
  214. if n is None:
  215. if default is not NO_DEFAULT:
  216. return default
  217. elif fatal:
  218. name = xpath if name is None else name
  219. raise ExtractorError(f'Could not find XML element {name}')
  220. else:
  221. return None
  222. return n
  223. def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
  224. n = xpath_element(node, xpath, name, fatal=fatal, default=default)
  225. if n is None or n == default:
  226. return n
  227. if n.text is None:
  228. if default is not NO_DEFAULT:
  229. return default
  230. elif fatal:
  231. name = xpath if name is None else name
  232. raise ExtractorError(f'Could not find XML element\'s text {name}')
  233. else:
  234. return None
  235. return n.text
  236. def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
  237. n = find_xpath_attr(node, xpath, key)
  238. if n is None:
  239. if default is not NO_DEFAULT:
  240. return default
  241. elif fatal:
  242. name = f'{xpath}[@{key}]' if name is None else name
  243. raise ExtractorError(f'Could not find XML attribute {name}')
  244. else:
  245. return None
  246. return n.attrib[key]
  247. def get_element_by_id(id, html, **kwargs):
  248. """Return the content of the tag with the specified ID in the passed HTML document"""
  249. return get_element_by_attribute('id', id, html, **kwargs)
  250. def get_element_html_by_id(id, html, **kwargs):
  251. """Return the html of the tag with the specified ID in the passed HTML document"""
  252. return get_element_html_by_attribute('id', id, html, **kwargs)
  253. def get_element_by_class(class_name, html):
  254. """Return the content of the first tag with the specified class in the passed HTML document"""
  255. retval = get_elements_by_class(class_name, html)
  256. return retval[0] if retval else None
  257. def get_element_html_by_class(class_name, html):
  258. """Return the html of the first tag with the specified class in the passed HTML document"""
  259. retval = get_elements_html_by_class(class_name, html)
  260. return retval[0] if retval else None
  261. def get_element_by_attribute(attribute, value, html, **kwargs):
  262. retval = get_elements_by_attribute(attribute, value, html, **kwargs)
  263. return retval[0] if retval else None
  264. def get_element_html_by_attribute(attribute, value, html, **kargs):
  265. retval = get_elements_html_by_attribute(attribute, value, html, **kargs)
  266. return retval[0] if retval else None
  267. def get_elements_by_class(class_name, html, **kargs):
  268. """Return the content of all tags with the specified class in the passed HTML document as a list"""
  269. return get_elements_by_attribute(
  270. 'class', rf'[^\'"]*(?<=[\'"\s]){re.escape(class_name)}(?=[\'"\s])[^\'"]*',
  271. html, escape_value=False)
  272. def get_elements_html_by_class(class_name, html):
  273. """Return the html of all tags with the specified class in the passed HTML document as a list"""
  274. return get_elements_html_by_attribute(
  275. 'class', rf'[^\'"]*(?<=[\'"\s]){re.escape(class_name)}(?=[\'"\s])[^\'"]*',
  276. html, escape_value=False)
  277. def get_elements_by_attribute(*args, **kwargs):
  278. """Return the content of the tag with the specified attribute in the passed HTML document"""
  279. return [content for content, _ in get_elements_text_and_html_by_attribute(*args, **kwargs)]
  280. def get_elements_html_by_attribute(*args, **kwargs):
  281. """Return the html of the tag with the specified attribute in the passed HTML document"""
  282. return [whole for _, whole in get_elements_text_and_html_by_attribute(*args, **kwargs)]
  283. def get_elements_text_and_html_by_attribute(attribute, value, html, *, tag=r'[\w:.-]+', escape_value=True):
  284. """
  285. Return the text (content) and the html (whole) of the tag with the specified
  286. attribute in the passed HTML document
  287. """
  288. if not value:
  289. return
  290. quote = '' if re.match(r'''[\s"'`=<>]''', value) else '?'
  291. value = re.escape(value) if escape_value else value
  292. partial_element_re = rf'''(?x)
  293. <(?P<tag>{tag})
  294. (?:\s(?:[^>"']|"[^"]*"|'[^']*')*)?
  295. \s{re.escape(attribute)}\s*=\s*(?P<_q>['"]{quote})(?-x:{value})(?P=_q)
  296. '''
  297. for m in re.finditer(partial_element_re, html):
  298. content, whole = get_element_text_and_html_by_tag(m.group('tag'), html[m.start():])
  299. yield (
  300. unescapeHTML(re.sub(r'^(?P<q>["\'])(?P<content>.*)(?P=q)$', r'\g<content>', content, flags=re.DOTALL)),
  301. whole,
  302. )
  303. class HTMLBreakOnClosingTagParser(html.parser.HTMLParser):
  304. """
  305. HTML parser which raises HTMLBreakOnClosingTagException upon reaching the
  306. closing tag for the first opening tag it has encountered, and can be used
  307. as a context manager
  308. """
  309. class HTMLBreakOnClosingTagException(Exception):
  310. pass
  311. def __init__(self):
  312. self.tagstack = collections.deque()
  313. html.parser.HTMLParser.__init__(self)
  314. def __enter__(self):
  315. return self
  316. def __exit__(self, *_):
  317. self.close()
  318. def close(self):
  319. # handle_endtag does not return upon raising HTMLBreakOnClosingTagException,
  320. # so data remains buffered; we no longer have any interest in it, thus
  321. # override this method to discard it
  322. pass
  323. def handle_starttag(self, tag, _):
  324. self.tagstack.append(tag)
  325. def handle_endtag(self, tag):
  326. if not self.tagstack:
  327. raise compat_HTMLParseError('no tags in the stack')
  328. while self.tagstack:
  329. inner_tag = self.tagstack.pop()
  330. if inner_tag == tag:
  331. break
  332. else:
  333. raise compat_HTMLParseError(f'matching opening tag for closing {tag} tag not found')
  334. if not self.tagstack:
  335. raise self.HTMLBreakOnClosingTagException
  336. # XXX: This should be far less strict
  337. def get_element_text_and_html_by_tag(tag, html):
  338. """
  339. For the first element with the specified tag in the passed HTML document
  340. return its' content (text) and the whole element (html)
  341. """
  342. def find_or_raise(haystack, needle, exc):
  343. try:
  344. return haystack.index(needle)
  345. except ValueError:
  346. raise exc
  347. closing_tag = f'</{tag}>'
  348. whole_start = find_or_raise(
  349. html, f'<{tag}', compat_HTMLParseError(f'opening {tag} tag not found'))
  350. content_start = find_or_raise(
  351. html[whole_start:], '>', compat_HTMLParseError(f'malformed opening {tag} tag'))
  352. content_start += whole_start + 1
  353. with HTMLBreakOnClosingTagParser() as parser:
  354. parser.feed(html[whole_start:content_start])
  355. if not parser.tagstack or parser.tagstack[0] != tag:
  356. raise compat_HTMLParseError(f'parser did not match opening {tag} tag')
  357. offset = content_start
  358. while offset < len(html):
  359. next_closing_tag_start = find_or_raise(
  360. html[offset:], closing_tag,
  361. compat_HTMLParseError(f'closing {tag} tag not found'))
  362. next_closing_tag_end = next_closing_tag_start + len(closing_tag)
  363. try:
  364. parser.feed(html[offset:offset + next_closing_tag_end])
  365. offset += next_closing_tag_end
  366. except HTMLBreakOnClosingTagParser.HTMLBreakOnClosingTagException:
  367. return html[content_start:offset + next_closing_tag_start], \
  368. html[whole_start:offset + next_closing_tag_end]
  369. raise compat_HTMLParseError('unexpected end of html')
  370. class HTMLAttributeParser(html.parser.HTMLParser):
  371. """Trivial HTML parser to gather the attributes for a single element"""
  372. def __init__(self):
  373. self.attrs = {}
  374. html.parser.HTMLParser.__init__(self)
  375. def handle_starttag(self, tag, attrs):
  376. self.attrs = dict(attrs)
  377. raise compat_HTMLParseError('done')
  378. class HTMLListAttrsParser(html.parser.HTMLParser):
  379. """HTML parser to gather the attributes for the elements of a list"""
  380. def __init__(self):
  381. html.parser.HTMLParser.__init__(self)
  382. self.items = []
  383. self._level = 0
  384. def handle_starttag(self, tag, attrs):
  385. if tag == 'li' and self._level == 0:
  386. self.items.append(dict(attrs))
  387. self._level += 1
  388. def handle_endtag(self, tag):
  389. self._level -= 1
  390. def extract_attributes(html_element):
  391. """Given a string for an HTML element such as
  392. <el
  393. a="foo" B="bar" c="&98;az" d=boz
  394. empty= noval entity="&amp;"
  395. sq='"' dq="'"
  396. >
  397. Decode and return a dictionary of attributes.
  398. {
  399. 'a': 'foo', 'b': 'bar', c: 'baz', d: 'boz',
  400. 'empty': '', 'noval': None, 'entity': '&',
  401. 'sq': '"', 'dq': '\''
  402. }.
  403. """
  404. parser = HTMLAttributeParser()
  405. with contextlib.suppress(compat_HTMLParseError):
  406. parser.feed(html_element)
  407. parser.close()
  408. return parser.attrs
  409. def parse_list(webpage):
  410. """Given a string for an series of HTML <li> elements,
  411. return a dictionary of their attributes"""
  412. parser = HTMLListAttrsParser()
  413. parser.feed(webpage)
  414. parser.close()
  415. return parser.items
  416. def clean_html(html):
  417. """Clean an HTML snippet into a readable string"""
  418. if html is None: # Convenience for sanitizing descriptions etc.
  419. return html
  420. html = re.sub(r'\s+', ' ', html)
  421. html = re.sub(r'(?u)\s?<\s?br\s?/?\s?>\s?', '\n', html)
  422. html = re.sub(r'(?u)<\s?/\s?p\s?>\s?<\s?p[^>]*>', '\n', html)
  423. # Strip html tags
  424. html = re.sub('<.*?>', '', html)
  425. # Replace html entities
  426. html = unescapeHTML(html)
  427. return html.strip()
  428. class LenientJSONDecoder(json.JSONDecoder):
  429. # TODO: Write tests
  430. def __init__(self, *args, transform_source=None, ignore_extra=False, close_objects=0, **kwargs):
  431. self.transform_source, self.ignore_extra = transform_source, ignore_extra
  432. self._close_attempts = 2 * close_objects
  433. super().__init__(*args, **kwargs)
  434. @staticmethod
  435. def _close_object(err):
  436. doc = err.doc[:err.pos]
  437. # We need to add comma first to get the correct error message
  438. if err.msg.startswith('Expecting \',\''):
  439. return doc + ','
  440. elif not doc.endswith(','):
  441. return
  442. if err.msg.startswith('Expecting property name'):
  443. return doc[:-1] + '}'
  444. elif err.msg.startswith('Expecting value'):
  445. return doc[:-1] + ']'
  446. def decode(self, s):
  447. if self.transform_source:
  448. s = self.transform_source(s)
  449. for attempt in range(self._close_attempts + 1):
  450. try:
  451. if self.ignore_extra:
  452. return self.raw_decode(s.lstrip())[0]
  453. return super().decode(s)
  454. except json.JSONDecodeError as e:
  455. if e.pos is None:
  456. raise
  457. elif attempt < self._close_attempts:
  458. s = self._close_object(e)
  459. if s is not None:
  460. continue
  461. raise type(e)(f'{e.msg} in {s[e.pos - 10:e.pos + 10]!r}', s, e.pos)
  462. assert False, 'Too many attempts to decode JSON'
  463. def sanitize_open(filename, open_mode):
  464. """Try to open the given filename, and slightly tweak it if this fails.
  465. Attempts to open the given filename. If this fails, it tries to change
  466. the filename slightly, step by step, until it's either able to open it
  467. or it fails and raises a final exception, like the standard open()
  468. function.
  469. It returns the tuple (stream, definitive_file_name).
  470. """
  471. if filename == '-':
  472. if sys.platform == 'win32':
  473. import msvcrt
  474. # stdout may be any IO stream, e.g. when using contextlib.redirect_stdout
  475. with contextlib.suppress(io.UnsupportedOperation):
  476. msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
  477. return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
  478. for attempt in range(2):
  479. try:
  480. try:
  481. if sys.platform == 'win32':
  482. # FIXME: An exclusive lock also locks the file from being read.
  483. # Since windows locks are mandatory, don't lock the file on windows (for now).
  484. # Ref: https://github.com/yt-dlp/yt-dlp/issues/3124
  485. raise LockingUnsupportedError
  486. stream = locked_file(filename, open_mode, block=False).__enter__()
  487. except OSError:
  488. stream = open(filename, open_mode)
  489. return stream, filename
  490. except OSError as err:
  491. if attempt or err.errno in (errno.EACCES,):
  492. raise
  493. old_filename, filename = filename, sanitize_path(filename)
  494. if old_filename == filename:
  495. raise
  496. def timeconvert(timestr):
  497. """Convert RFC 2822 defined time string into system timestamp"""
  498. timestamp = None
  499. timetuple = email.utils.parsedate_tz(timestr)
  500. if timetuple is not None:
  501. timestamp = email.utils.mktime_tz(timetuple)
  502. return timestamp
  503. def sanitize_filename(s, restricted=False, is_id=NO_DEFAULT):
  504. """Sanitizes a string so it could be used as part of a filename.
  505. @param restricted Use a stricter subset of allowed characters
  506. @param is_id Whether this is an ID that should be kept unchanged if possible.
  507. If unset, yt-dlp's new sanitization rules are in effect
  508. """
  509. if s == '':
  510. return ''
  511. def replace_insane(char):
  512. if restricted and char in ACCENT_CHARS:
  513. return ACCENT_CHARS[char]
  514. elif not restricted and char == '\n':
  515. return '\0 '
  516. elif is_id is NO_DEFAULT and not restricted and char in '"*:<>?|/\\':
  517. # Replace with their full-width unicode counterparts
  518. return {'/': '\u29F8', '\\': '\u29f9'}.get(char, chr(ord(char) + 0xfee0))
  519. elif char == '?' or ord(char) < 32 or ord(char) == 127:
  520. return ''
  521. elif char == '"':
  522. return '' if restricted else '\''
  523. elif char == ':':
  524. return '\0_\0-' if restricted else '\0 \0-'
  525. elif char in '\\/|*<>':
  526. return '\0_'
  527. if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace() or ord(char) > 127):
  528. return '' if unicodedata.category(char)[0] in 'CM' else '\0_'
  529. return char
  530. # Replace look-alike Unicode glyphs
  531. if restricted and (is_id is NO_DEFAULT or not is_id):
  532. s = unicodedata.normalize('NFKC', s)
  533. s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s) # Handle timestamps
  534. result = ''.join(map(replace_insane, s))
  535. if is_id is NO_DEFAULT:
  536. result = re.sub(r'(\0.)(?:(?=\1)..)+', r'\1', result) # Remove repeated substitute chars
  537. STRIP_RE = r'(?:\0.|[ _-])*'
  538. result = re.sub(f'^\0.{STRIP_RE}|{STRIP_RE}\0.$', '', result) # Remove substitute chars from start/end
  539. result = result.replace('\0', '') or '_'
  540. if not is_id:
  541. while '__' in result:
  542. result = result.replace('__', '_')
  543. result = result.strip('_')
  544. # Common case of "Foreign band name - English song title"
  545. if restricted and result.startswith('-_'):
  546. result = result[2:]
  547. if result.startswith('-'):
  548. result = '_' + result[len('-'):]
  549. result = result.lstrip('.')
  550. if not result:
  551. result = '_'
  552. return result
  553. def sanitize_path(s, force=False):
  554. """Sanitizes and normalizes path on Windows"""
  555. # XXX: this handles drive relative paths (c:sth) incorrectly
  556. if sys.platform == 'win32':
  557. force = False
  558. drive_or_unc, _ = os.path.splitdrive(s)
  559. elif force:
  560. drive_or_unc = ''
  561. else:
  562. return s
  563. norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
  564. if drive_or_unc:
  565. norm_path.pop(0)
  566. sanitized_path = [
  567. path_part if path_part in ['.', '..'] else re.sub(r'(?:[/<>:"\|\\?\*]|[\s.]$)', '#', path_part)
  568. for path_part in norm_path]
  569. if drive_or_unc:
  570. sanitized_path.insert(0, drive_or_unc + os.path.sep)
  571. elif force and s and s[0] == os.path.sep:
  572. sanitized_path.insert(0, os.path.sep)
  573. # TODO: Fix behavioral differences <3.12
  574. # The workaround using `normpath` only superficially passes tests
  575. # Ref: https://github.com/python/cpython/pull/100351
  576. return os.path.normpath(os.path.join(*sanitized_path))
  577. def sanitize_url(url, *, scheme='http'):
  578. # Prepend protocol-less URLs with `http:` scheme in order to mitigate
  579. # the number of unwanted failures due to missing protocol
  580. if url is None:
  581. return
  582. elif url.startswith('//'):
  583. return f'{scheme}:{url}'
  584. # Fix some common typos seen so far
  585. COMMON_TYPOS = (
  586. # https://github.com/ytdl-org/youtube-dl/issues/15649
  587. (r'^httpss://', r'https://'),
  588. # https://bx1.be/lives/direct-tv/
  589. (r'^rmtp([es]?)://', r'rtmp\1://'),
  590. )
  591. for mistake, fixup in COMMON_TYPOS:
  592. if re.match(mistake, url):
  593. return re.sub(mistake, fixup, url)
  594. return url
  595. def extract_basic_auth(url):
  596. parts = urllib.parse.urlsplit(url)
  597. if parts.username is None:
  598. return url, None
  599. url = urllib.parse.urlunsplit(parts._replace(netloc=(
  600. parts.hostname if parts.port is None
  601. else f'{parts.hostname}:{parts.port}')))
  602. auth_payload = base64.b64encode(
  603. ('{}:{}'.format(parts.username, parts.password or '')).encode())
  604. return url, f'Basic {auth_payload.decode()}'
  605. def expand_path(s):
  606. """Expand shell variables and ~"""
  607. return os.path.expandvars(compat_expanduser(s))
  608. def orderedSet(iterable, *, lazy=False):
  609. """Remove all duplicates from the input iterable"""
  610. def _iter():
  611. seen = [] # Do not use set since the items can be unhashable
  612. for x in iterable:
  613. if x not in seen:
  614. seen.append(x)
  615. yield x
  616. return _iter() if lazy else list(_iter())
  617. def _htmlentity_transform(entity_with_semicolon):
  618. """Transforms an HTML entity to a character."""
  619. entity = entity_with_semicolon[:-1]
  620. # Known non-numeric HTML entity
  621. if entity in html.entities.name2codepoint:
  622. return chr(html.entities.name2codepoint[entity])
  623. # TODO: HTML5 allows entities without a semicolon.
  624. # E.g. '&Eacuteric' should be decoded as 'Éric'.
  625. if entity_with_semicolon in html.entities.html5:
  626. return html.entities.html5[entity_with_semicolon]
  627. mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
  628. if mobj is not None:
  629. numstr = mobj.group(1)
  630. if numstr.startswith('x'):
  631. base = 16
  632. numstr = f'0{numstr}'
  633. else:
  634. base = 10
  635. # See https://github.com/ytdl-org/youtube-dl/issues/7518
  636. with contextlib.suppress(ValueError):
  637. return chr(int(numstr, base))
  638. # Unknown entity in name, return its literal representation
  639. return f'&{entity};'
  640. def unescapeHTML(s):
  641. if s is None:
  642. return None
  643. assert isinstance(s, str)
  644. return re.sub(
  645. r'&([^&;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
  646. def escapeHTML(text):
  647. return (
  648. text
  649. .replace('&', '&amp;')
  650. .replace('<', '&lt;')
  651. .replace('>', '&gt;')
  652. .replace('"', '&quot;')
  653. .replace("'", '&#39;')
  654. )
  655. class netrc_from_content(netrc.netrc):
  656. def __init__(self, content):
  657. self.hosts, self.macros = {}, {}
  658. with io.StringIO(content) as stream:
  659. self._parse('-', stream, False)
  660. class Popen(subprocess.Popen):
  661. if sys.platform == 'win32':
  662. _startupinfo = subprocess.STARTUPINFO()
  663. _startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
  664. else:
  665. _startupinfo = None
  666. @staticmethod
  667. def _fix_pyinstaller_ld_path(env):
  668. """Restore LD_LIBRARY_PATH when using PyInstaller
  669. Ref: https://github.com/pyinstaller/pyinstaller/blob/develop/doc/runtime-information.rst#ld_library_path--libpath-considerations
  670. https://github.com/yt-dlp/yt-dlp/issues/4573
  671. """
  672. if not hasattr(sys, '_MEIPASS'):
  673. return
  674. def _fix(key):
  675. orig = env.get(f'{key}_ORIG')
  676. if orig is None:
  677. env.pop(key, None)
  678. else:
  679. env[key] = orig
  680. _fix('LD_LIBRARY_PATH') # Linux
  681. _fix('DYLD_LIBRARY_PATH') # macOS
  682. def __init__(self, args, *remaining, env=None, text=False, shell=False, **kwargs):
  683. if env is None:
  684. env = os.environ.copy()
  685. self._fix_pyinstaller_ld_path(env)
  686. self.__text_mode = kwargs.get('encoding') or kwargs.get('errors') or text or kwargs.get('universal_newlines')
  687. if text is True:
  688. kwargs['universal_newlines'] = True # For 3.6 compatibility
  689. kwargs.setdefault('encoding', 'utf-8')
  690. kwargs.setdefault('errors', 'replace')
  691. if shell and compat_os_name == 'nt' and kwargs.get('executable') is None:
  692. if not isinstance(args, str):
  693. args = shell_quote(args, shell=True)
  694. shell = False
  695. # Set variable for `cmd.exe` newline escaping (see `utils.shell_quote`)
  696. env['='] = '"^\n\n"'
  697. args = f'{self.__comspec()} /Q /S /D /V:OFF /E:ON /C "{args}"'
  698. super().__init__(args, *remaining, env=env, shell=shell, **kwargs, startupinfo=self._startupinfo)
  699. def __comspec(self):
  700. comspec = os.environ.get('ComSpec') or os.path.join(
  701. os.environ.get('SystemRoot', ''), 'System32', 'cmd.exe')
  702. if os.path.isabs(comspec):
  703. return comspec
  704. raise FileNotFoundError('shell not found: neither %ComSpec% nor %SystemRoot% is set')
  705. def communicate_or_kill(self, *args, **kwargs):
  706. try:
  707. return self.communicate(*args, **kwargs)
  708. except BaseException: # Including KeyboardInterrupt
  709. self.kill(timeout=None)
  710. raise
  711. def kill(self, *, timeout=0):
  712. super().kill()
  713. if timeout != 0:
  714. self.wait(timeout=timeout)
  715. @classmethod
  716. def run(cls, *args, timeout=None, **kwargs):
  717. with cls(*args, **kwargs) as proc:
  718. default = '' if proc.__text_mode else b''
  719. stdout, stderr = proc.communicate_or_kill(timeout=timeout)
  720. return stdout or default, stderr or default, proc.returncode
  721. def encodeArgument(s):
  722. # Legacy code that uses byte strings
  723. # Uncomment the following line after fixing all post processors
  724. # assert isinstance(s, str), 'Internal error: %r should be of type %r, is %r' % (s, str, type(s))
  725. return s if isinstance(s, str) else s.decode('ascii')
  726. _timetuple = collections.namedtuple('Time', ('hours', 'minutes', 'seconds', 'milliseconds'))
  727. def timetuple_from_msec(msec):
  728. secs, msec = divmod(msec, 1000)
  729. mins, secs = divmod(secs, 60)
  730. hrs, mins = divmod(mins, 60)
  731. return _timetuple(hrs, mins, secs, msec)
  732. def formatSeconds(secs, delim=':', msec=False):
  733. time = timetuple_from_msec(secs * 1000)
  734. if time.hours:
  735. ret = '%d%s%02d%s%02d' % (time.hours, delim, time.minutes, delim, time.seconds)
  736. elif time.minutes:
  737. ret = '%d%s%02d' % (time.minutes, delim, time.seconds)
  738. else:
  739. ret = '%d' % time.seconds
  740. return '%s.%03d' % (ret, time.milliseconds) if msec else ret
  741. def bug_reports_message(before=';'):
  742. from ..update import REPOSITORY
  743. msg = (f'please report this issue on https://github.com/{REPOSITORY}/issues?q= , '
  744. 'filling out the appropriate issue template. Confirm you are on the latest version using yt-dlp -U')
  745. before = before.rstrip()
  746. if not before or before.endswith(('.', '!', '?')):
  747. msg = msg[0].title() + msg[1:]
  748. return (before + ' ' if before else '') + msg
  749. class YoutubeDLError(Exception):
  750. """Base exception for YoutubeDL errors."""
  751. msg = None
  752. def __init__(self, msg=None):
  753. if msg is not None:
  754. self.msg = msg
  755. elif self.msg is None:
  756. self.msg = type(self).__name__
  757. super().__init__(self.msg)
  758. class ExtractorError(YoutubeDLError):
  759. """Error during info extraction."""
  760. def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None, ie=None):
  761. """ tb, if given, is the original traceback (so that it can be printed out).
  762. If expected is set, this is a normal error message and most likely not a bug in yt-dlp.
  763. """
  764. from ..networking.exceptions import network_exceptions
  765. if sys.exc_info()[0] in network_exceptions:
  766. expected = True
  767. self.orig_msg = str(msg)
  768. self.traceback = tb
  769. self.expected = expected
  770. self.cause = cause
  771. self.video_id = video_id
  772. self.ie = ie
  773. self.exc_info = sys.exc_info() # preserve original exception
  774. if isinstance(self.exc_info[1], ExtractorError):
  775. self.exc_info = self.exc_info[1].exc_info
  776. super().__init__(self.__msg)
  777. @property
  778. def __msg(self):
  779. return ''.join((
  780. format_field(self.ie, None, '[%s] '),
  781. format_field(self.video_id, None, '%s: '),
  782. self.orig_msg,
  783. format_field(self.cause, None, ' (caused by %r)'),
  784. '' if self.expected else bug_reports_message()))
  785. def format_traceback(self):
  786. return join_nonempty(
  787. self.traceback and ''.join(traceback.format_tb(self.traceback)),
  788. self.cause and ''.join(traceback.format_exception(None, self.cause, self.cause.__traceback__)[1:]),
  789. delim='\n') or None
  790. def __setattr__(self, name, value):
  791. super().__setattr__(name, value)
  792. if getattr(self, 'msg', None) and name not in ('msg', 'args'):
  793. self.msg = self.__msg or type(self).__name__
  794. self.args = (self.msg, ) # Cannot be property
  795. class UnsupportedError(ExtractorError):
  796. def __init__(self, url):
  797. super().__init__(
  798. f'Unsupported URL: {url}', expected=True)
  799. self.url = url
  800. class RegexNotFoundError(ExtractorError):
  801. """Error when a regex didn't match"""
  802. pass
  803. class GeoRestrictedError(ExtractorError):
  804. """Geographic restriction Error exception.
  805. This exception may be thrown when a video is not available from your
  806. geographic location due to geographic restrictions imposed by a website.
  807. """
  808. def __init__(self, msg, countries=None, **kwargs):
  809. kwargs['expected'] = True
  810. super().__init__(msg, **kwargs)
  811. self.countries = countries
  812. class UserNotLive(ExtractorError):
  813. """Error when a channel/user is not live"""
  814. def __init__(self, msg=None, **kwargs):
  815. kwargs['expected'] = True
  816. super().__init__(msg or 'The channel is not currently live', **kwargs)
  817. class DownloadError(YoutubeDLError):
  818. """Download Error exception.
  819. This exception may be thrown by FileDownloader objects if they are not
  820. configured to continue on errors. They will contain the appropriate
  821. error message.
  822. """
  823. def __init__(self, msg, exc_info=None):
  824. """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
  825. super().__init__(msg)
  826. self.exc_info = exc_info
  827. class EntryNotInPlaylist(YoutubeDLError):
  828. """Entry not in playlist exception.
  829. This exception will be thrown by YoutubeDL when a requested entry
  830. is not found in the playlist info_dict
  831. """
  832. msg = 'Entry not found in info'
  833. class SameFileError(YoutubeDLError):
  834. """Same File exception.
  835. This exception will be thrown by FileDownloader objects if they detect
  836. multiple files would have to be downloaded to the same file on disk.
  837. """
  838. msg = 'Fixed output name but more than one file to download'
  839. def __init__(self, filename=None):
  840. if filename is not None:
  841. self.msg += f': {filename}'
  842. super().__init__(self.msg)
  843. class PostProcessingError(YoutubeDLError):
  844. """Post Processing exception.
  845. This exception may be raised by PostProcessor's .run() method to
  846. indicate an error in the postprocessing task.
  847. """
  848. class DownloadCancelled(YoutubeDLError):
  849. """ Exception raised when the download queue should be interrupted """
  850. msg = 'The download was cancelled'
  851. class ExistingVideoReached(DownloadCancelled):
  852. """ --break-on-existing triggered """
  853. msg = 'Encountered a video that is already in the archive, stopping due to --break-on-existing'
  854. class RejectedVideoReached(DownloadCancelled):
  855. """ --break-match-filter triggered """
  856. msg = 'Encountered a video that did not match filter, stopping due to --break-match-filter'
  857. class MaxDownloadsReached(DownloadCancelled):
  858. """ --max-downloads limit has been reached. """
  859. msg = 'Maximum number of downloads reached, stopping due to --max-downloads'
  860. class ReExtractInfo(YoutubeDLError):
  861. """ Video info needs to be re-extracted. """
  862. def __init__(self, msg, expected=False):
  863. super().__init__(msg)
  864. self.expected = expected
  865. class ThrottledDownload(ReExtractInfo):
  866. """ Download speed below --throttled-rate. """
  867. msg = 'The download speed is below throttle limit'
  868. def __init__(self):
  869. super().__init__(self.msg, expected=False)
  870. class UnavailableVideoError(YoutubeDLError):
  871. """Unavailable Format exception.
  872. This exception will be thrown when a video is requested
  873. in a format that is not available for that video.
  874. """
  875. msg = 'Unable to download video'
  876. def __init__(self, err=None):
  877. if err is not None:
  878. self.msg += f': {err}'
  879. super().__init__(self.msg)
  880. class ContentTooShortError(YoutubeDLError):
  881. """Content Too Short exception.
  882. This exception may be raised by FileDownloader objects when a file they
  883. download is too small for what the server announced first, indicating
  884. the connection was probably interrupted.
  885. """
  886. def __init__(self, downloaded, expected):
  887. super().__init__(f'Downloaded {downloaded} bytes, expected {expected} bytes')
  888. # Both in bytes
  889. self.downloaded = downloaded
  890. self.expected = expected
  891. class XAttrMetadataError(YoutubeDLError):
  892. def __init__(self, code=None, msg='Unknown error'):
  893. super().__init__(msg)
  894. self.code = code
  895. self.msg = msg
  896. # Parsing code and msg
  897. if (self.code in (errno.ENOSPC, errno.EDQUOT)
  898. or 'No space left' in self.msg or 'Disk quota exceeded' in self.msg):
  899. self.reason = 'NO_SPACE'
  900. elif self.code == errno.E2BIG or 'Argument list too long' in self.msg:
  901. self.reason = 'VALUE_TOO_LONG'
  902. else:
  903. self.reason = 'NOT_SUPPORTED'
  904. class XAttrUnavailableError(YoutubeDLError):
  905. pass
  906. def is_path_like(f):
  907. return isinstance(f, (str, bytes, os.PathLike))
  908. def extract_timezone(date_str, default=None):
  909. m = re.search(
  910. r'''(?x)
  911. ^.{8,}? # >=8 char non-TZ prefix, if present
  912. (?P<tz>Z| # just the UTC Z, or
  913. (?:(?<=.\b\d{4}|\b\d{2}:\d\d)| # preceded by 4 digits or hh:mm or
  914. (?<!.\b[a-zA-Z]{3}|[a-zA-Z]{4}|..\b\d\d)) # not preceded by 3 alpha word or >= 4 alpha or 2 digits
  915. [ ]? # optional space
  916. (?P<sign>\+|-) # +/-
  917. (?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2}) # hh[:]mm
  918. $)
  919. ''', date_str)
  920. timezone = None
  921. if not m:
  922. m = re.search(r'\d{1,2}:\d{1,2}(?:\.\d+)?(?P<tz>\s*[A-Z]+)$', date_str)
  923. timezone = TIMEZONE_NAMES.get(m and m.group('tz').strip())
  924. if timezone is not None:
  925. date_str = date_str[:-len(m.group('tz'))]
  926. timezone = dt.timedelta(hours=timezone)
  927. else:
  928. date_str = date_str[:-len(m.group('tz'))]
  929. if m.group('sign'):
  930. sign = 1 if m.group('sign') == '+' else -1
  931. timezone = dt.timedelta(
  932. hours=sign * int(m.group('hours')),
  933. minutes=sign * int(m.group('minutes')))
  934. if timezone is None and default is not NO_DEFAULT:
  935. timezone = default or dt.timedelta()
  936. return timezone, date_str
  937. def parse_iso8601(date_str, delimiter='T', timezone=None):
  938. """ Return a UNIX timestamp from the given date """
  939. if date_str is None:
  940. return None
  941. date_str = re.sub(r'\.[0-9]+', '', date_str)
  942. timezone, date_str = extract_timezone(date_str, timezone)
  943. with contextlib.suppress(ValueError, TypeError):
  944. date_format = f'%Y-%m-%d{delimiter}%H:%M:%S'
  945. dt_ = dt.datetime.strptime(date_str, date_format) - timezone
  946. return calendar.timegm(dt_.timetuple())
  947. def date_formats(day_first=True):
  948. return DATE_FORMATS_DAY_FIRST if day_first else DATE_FORMATS_MONTH_FIRST
  949. def unified_strdate(date_str, day_first=True):
  950. """Return a string with the date in the format YYYYMMDD"""
  951. if date_str is None:
  952. return None
  953. upload_date = None
  954. # Replace commas
  955. date_str = date_str.replace(',', ' ')
  956. # Remove AM/PM + timezone
  957. date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
  958. _, date_str = extract_timezone(date_str)
  959. for expression in date_formats(day_first):
  960. with contextlib.suppress(ValueError):
  961. upload_date = dt.datetime.strptime(date_str, expression).strftime('%Y%m%d')
  962. if upload_date is None:
  963. timetuple = email.utils.parsedate_tz(date_str)
  964. if timetuple:
  965. with contextlib.suppress(ValueError):
  966. upload_date = dt.datetime(*timetuple[:6]).strftime('%Y%m%d')
  967. if upload_date is not None:
  968. return str(upload_date)
  969. def unified_timestamp(date_str, day_first=True):
  970. if not isinstance(date_str, str):
  971. return None
  972. date_str = re.sub(r'\s+', ' ', re.sub(
  973. r'(?i)[,|]|(mon|tues?|wed(nes)?|thu(rs)?|fri|sat(ur)?|sun)(day)?', '', date_str))
  974. pm_delta = 12 if re.search(r'(?i)PM', date_str) else 0
  975. timezone, date_str = extract_timezone(date_str)
  976. # Remove AM/PM + timezone
  977. date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
  978. # Remove unrecognized timezones from ISO 8601 alike timestamps
  979. m = re.search(r'\d{1,2}:\d{1,2}(?:\.\d+)?(?P<tz>\s*[A-Z]+)$', date_str)
  980. if m:
  981. date_str = date_str[:-len(m.group('tz'))]
  982. # Python only supports microseconds, so remove nanoseconds
  983. m = re.search(r'^([0-9]{4,}-[0-9]{1,2}-[0-9]{1,2}T[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.[0-9]{6})[0-9]+$', date_str)
  984. if m:
  985. date_str = m.group(1)
  986. for expression in date_formats(day_first):
  987. with contextlib.suppress(ValueError):
  988. dt_ = dt.datetime.strptime(date_str, expression) - timezone + dt.timedelta(hours=pm_delta)
  989. return calendar.timegm(dt_.timetuple())
  990. timetuple = email.utils.parsedate_tz(date_str)
  991. if timetuple:
  992. return calendar.timegm(timetuple) + pm_delta * 3600 - timezone.total_seconds()
  993. def determine_ext(url, default_ext='unknown_video'):
  994. if url is None or '.' not in url:
  995. return default_ext
  996. guess = url.partition('?')[0].rpartition('.')[2]
  997. if re.match(r'^[A-Za-z0-9]+$', guess):
  998. return guess
  999. # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
  1000. elif guess.rstrip('/') in KNOWN_EXTENSIONS:
  1001. return guess.rstrip('/')
  1002. else:
  1003. return default_ext
  1004. def subtitles_filename(filename, sub_lang, sub_format, expected_real_ext=None):
  1005. return replace_extension(filename, sub_lang + '.' + sub_format, expected_real_ext)
  1006. def datetime_from_str(date_str, precision='auto', format='%Y%m%d'):
  1007. R"""
  1008. Return a datetime object from a string.
  1009. Supported format:
  1010. (now|today|yesterday|DATE)([+-]\d+(microsecond|second|minute|hour|day|week|month|year)s?)?
  1011. @param format strftime format of DATE
  1012. @param precision Round the datetime object: auto|microsecond|second|minute|hour|day
  1013. auto: round to the unit provided in date_str (if applicable).
  1014. """
  1015. auto_precision = False
  1016. if precision == 'auto':
  1017. auto_precision = True
  1018. precision = 'microsecond'
  1019. today = datetime_round(dt.datetime.now(dt.timezone.utc), precision)
  1020. if date_str in ('now', 'today'):
  1021. return today
  1022. if date_str == 'yesterday':
  1023. return today - dt.timedelta(days=1)
  1024. match = re.match(
  1025. r'(?P<start>.+)(?P<sign>[+-])(?P<time>\d+)(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?',
  1026. date_str)
  1027. if match is not None:
  1028. start_time = datetime_from_str(match.group('start'), precision, format)
  1029. time = int(match.group('time')) * (-1 if match.group('sign') == '-' else 1)
  1030. unit = match.group('unit')
  1031. if unit == 'month' or unit == 'year':
  1032. new_date = datetime_add_months(start_time, time * 12 if unit == 'year' else time)
  1033. unit = 'day'
  1034. else:
  1035. if unit == 'week':
  1036. unit = 'day'
  1037. time *= 7
  1038. delta = dt.timedelta(**{unit + 's': time})
  1039. new_date = start_time + delta
  1040. if auto_precision:
  1041. return datetime_round(new_date, unit)
  1042. return new_date
  1043. return datetime_round(dt.datetime.strptime(date_str, format), precision)
  1044. def date_from_str(date_str, format='%Y%m%d', strict=False):
  1045. R"""
  1046. Return a date object from a string using datetime_from_str
  1047. @param strict Restrict allowed patterns to "YYYYMMDD" and
  1048. (now|today|yesterday)(-\d+(day|week|month|year)s?)?
  1049. """
  1050. if strict and not re.fullmatch(r'\d{8}|(now|today|yesterday)(-\d+(day|week|month|year)s?)?', date_str):
  1051. raise ValueError(f'Invalid date format "{date_str}"')
  1052. return datetime_from_str(date_str, precision='microsecond', format=format).date()
  1053. def datetime_add_months(dt_, months):
  1054. """Increment/Decrement a datetime object by months."""
  1055. month = dt_.month + months - 1
  1056. year = dt_.year + month // 12
  1057. month = month % 12 + 1
  1058. day = min(dt_.day, calendar.monthrange(year, month)[1])
  1059. return dt_.replace(year, month, day)
  1060. def datetime_round(dt_, precision='day'):
  1061. """
  1062. Round a datetime object's time to a specific precision
  1063. """
  1064. if precision == 'microsecond':
  1065. return dt_
  1066. unit_seconds = {
  1067. 'day': 86400,
  1068. 'hour': 3600,
  1069. 'minute': 60,
  1070. 'second': 1,
  1071. }
  1072. roundto = lambda x, n: ((x + n / 2) // n) * n
  1073. timestamp = roundto(calendar.timegm(dt_.timetuple()), unit_seconds[precision])
  1074. return dt.datetime.fromtimestamp(timestamp, dt.timezone.utc)
  1075. def hyphenate_date(date_str):
  1076. """
  1077. Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
  1078. match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
  1079. if match is not None:
  1080. return '-'.join(match.groups())
  1081. else:
  1082. return date_str
  1083. class DateRange:
  1084. """Represents a time interval between two dates"""
  1085. def __init__(self, start=None, end=None):
  1086. """start and end must be strings in the format accepted by date"""
  1087. if start is not None:
  1088. self.start = date_from_str(start, strict=True)
  1089. else:
  1090. self.start = dt.datetime.min.date()
  1091. if end is not None:
  1092. self.end = date_from_str(end, strict=True)
  1093. else:
  1094. self.end = dt.datetime.max.date()
  1095. if self.start > self.end:
  1096. raise ValueError(f'Date range: "{self}" , the start date must be before the end date')
  1097. @classmethod
  1098. def day(cls, day):
  1099. """Returns a range that only contains the given day"""
  1100. return cls(day, day)
  1101. def __contains__(self, date):
  1102. """Check if the date is in the range"""
  1103. if not isinstance(date, dt.date):
  1104. date = date_from_str(date)
  1105. return self.start <= date <= self.end
  1106. def __repr__(self):
  1107. return f'{__name__}.{type(self).__name__}({self.start.isoformat()!r}, {self.end.isoformat()!r})'
  1108. def __str__(self):
  1109. return f'{self.start} to {self.end}'
  1110. def __eq__(self, other):
  1111. return (isinstance(other, DateRange)
  1112. and self.start == other.start and self.end == other.end)
  1113. @functools.cache
  1114. def system_identifier():
  1115. python_implementation = platform.python_implementation()
  1116. if python_implementation == 'PyPy' and hasattr(sys, 'pypy_version_info'):
  1117. python_implementation += ' version %d.%d.%d' % sys.pypy_version_info[:3]
  1118. libc_ver = []
  1119. with contextlib.suppress(OSError): # We may not have access to the executable
  1120. libc_ver = platform.libc_ver()
  1121. return 'Python {} ({} {} {}) - {} ({}{})'.format(
  1122. platform.python_version(),
  1123. python_implementation,
  1124. platform.machine(),
  1125. platform.architecture()[0],
  1126. platform.platform(),
  1127. ssl.OPENSSL_VERSION,
  1128. format_field(join_nonempty(*libc_ver, delim=' '), None, ', %s'),
  1129. )
  1130. @functools.cache
  1131. def get_windows_version():
  1132. """ Get Windows version. returns () if it's not running on Windows """
  1133. if compat_os_name == 'nt':
  1134. return version_tuple(platform.win32_ver()[1])
  1135. else:
  1136. return ()
  1137. def write_string(s, out=None, encoding=None):
  1138. assert isinstance(s, str)
  1139. out = out or sys.stderr
  1140. # `sys.stderr` might be `None` (Ref: https://github.com/pyinstaller/pyinstaller/pull/7217)
  1141. if not out:
  1142. return
  1143. if compat_os_name == 'nt' and supports_terminal_sequences(out):
  1144. s = re.sub(r'([\r\n]+)', r' \1', s)
  1145. enc, buffer = None, out
  1146. # `mode` might be `None` (Ref: https://github.com/yt-dlp/yt-dlp/issues/8816)
  1147. if 'b' in (getattr(out, 'mode', None) or ''):
  1148. enc = encoding or preferredencoding()
  1149. elif hasattr(out, 'buffer'):
  1150. buffer = out.buffer
  1151. enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
  1152. buffer.write(s.encode(enc, 'ignore') if enc else s)
  1153. out.flush()
  1154. # TODO: Use global logger
  1155. def deprecation_warning(msg, *, printer=None, stacklevel=0, **kwargs):
  1156. from .. import _IN_CLI
  1157. if _IN_CLI:
  1158. if msg in deprecation_warning._cache:
  1159. return
  1160. deprecation_warning._cache.add(msg)
  1161. if printer:
  1162. return printer(f'{msg}{bug_reports_message()}', **kwargs)
  1163. return write_string(f'ERROR: {msg}{bug_reports_message()}\n', **kwargs)
  1164. else:
  1165. import warnings
  1166. warnings.warn(DeprecationWarning(msg), stacklevel=stacklevel + 3)
  1167. deprecation_warning._cache = set()
  1168. def bytes_to_intlist(bs):
  1169. if not bs:
  1170. return []
  1171. if isinstance(bs[0], int): # Python 3
  1172. return list(bs)
  1173. else:
  1174. return [ord(c) for c in bs]
  1175. def intlist_to_bytes(xs):
  1176. if not xs:
  1177. return b''
  1178. return struct.pack('%dB' % len(xs), *xs)
  1179. class LockingUnsupportedError(OSError):
  1180. msg = 'File locking is not supported'
  1181. def __init__(self):
  1182. super().__init__(self.msg)
  1183. # Cross-platform file locking
  1184. if sys.platform == 'win32':
  1185. import ctypes
  1186. import ctypes.wintypes
  1187. import msvcrt
  1188. class OVERLAPPED(ctypes.Structure):
  1189. _fields_ = [
  1190. ('Internal', ctypes.wintypes.LPVOID),
  1191. ('InternalHigh', ctypes.wintypes.LPVOID),
  1192. ('Offset', ctypes.wintypes.DWORD),
  1193. ('OffsetHigh', ctypes.wintypes.DWORD),
  1194. ('hEvent', ctypes.wintypes.HANDLE),
  1195. ]
  1196. kernel32 = ctypes.WinDLL('kernel32')
  1197. LockFileEx = kernel32.LockFileEx
  1198. LockFileEx.argtypes = [
  1199. ctypes.wintypes.HANDLE, # hFile
  1200. ctypes.wintypes.DWORD, # dwFlags
  1201. ctypes.wintypes.DWORD, # dwReserved
  1202. ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
  1203. ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
  1204. ctypes.POINTER(OVERLAPPED), # Overlapped
  1205. ]
  1206. LockFileEx.restype = ctypes.wintypes.BOOL
  1207. UnlockFileEx = kernel32.UnlockFileEx
  1208. UnlockFileEx.argtypes = [
  1209. ctypes.wintypes.HANDLE, # hFile
  1210. ctypes.wintypes.DWORD, # dwReserved
  1211. ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
  1212. ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
  1213. ctypes.POINTER(OVERLAPPED), # Overlapped
  1214. ]
  1215. UnlockFileEx.restype = ctypes.wintypes.BOOL
  1216. whole_low = 0xffffffff
  1217. whole_high = 0x7fffffff
  1218. def _lock_file(f, exclusive, block):
  1219. overlapped = OVERLAPPED()
  1220. overlapped.Offset = 0
  1221. overlapped.OffsetHigh = 0
  1222. overlapped.hEvent = 0
  1223. f._lock_file_overlapped_p = ctypes.pointer(overlapped)
  1224. if not LockFileEx(msvcrt.get_osfhandle(f.fileno()),
  1225. (0x2 if exclusive else 0x0) | (0x0 if block else 0x1),
  1226. 0, whole_low, whole_high, f._lock_file_overlapped_p):
  1227. # NB: No argument form of "ctypes.FormatError" does not work on PyPy
  1228. raise BlockingIOError(f'Locking file failed: {ctypes.FormatError(ctypes.GetLastError())!r}')
  1229. def _unlock_file(f):
  1230. assert f._lock_file_overlapped_p
  1231. handle = msvcrt.get_osfhandle(f.fileno())
  1232. if not UnlockFileEx(handle, 0, whole_low, whole_high, f._lock_file_overlapped_p):
  1233. raise OSError(f'Unlocking file failed: {ctypes.FormatError()!r}')
  1234. else:
  1235. try:
  1236. import fcntl
  1237. def _lock_file(f, exclusive, block):
  1238. flags = fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH
  1239. if not block:
  1240. flags |= fcntl.LOCK_NB
  1241. try:
  1242. fcntl.flock(f, flags)
  1243. except BlockingIOError:
  1244. raise
  1245. except OSError: # AOSP does not have flock()
  1246. fcntl.lockf(f, flags)
  1247. def _unlock_file(f):
  1248. with contextlib.suppress(OSError):
  1249. return fcntl.flock(f, fcntl.LOCK_UN)
  1250. with contextlib.suppress(OSError):
  1251. return fcntl.lockf(f, fcntl.LOCK_UN) # AOSP does not have flock()
  1252. return fcntl.flock(f, fcntl.LOCK_UN | fcntl.LOCK_NB) # virtiofs needs LOCK_NB on unlocking
  1253. except ImportError:
  1254. def _lock_file(f, exclusive, block):
  1255. raise LockingUnsupportedError
  1256. def _unlock_file(f):
  1257. raise LockingUnsupportedError
  1258. class locked_file:
  1259. locked = False
  1260. def __init__(self, filename, mode, block=True, encoding=None):
  1261. if mode not in {'r', 'rb', 'a', 'ab', 'w', 'wb'}:
  1262. raise NotImplementedError(mode)
  1263. self.mode, self.block = mode, block
  1264. writable = any(f in mode for f in 'wax+')
  1265. readable = any(f in mode for f in 'r+')
  1266. flags = functools.reduce(operator.ior, (
  1267. getattr(os, 'O_CLOEXEC', 0), # UNIX only
  1268. getattr(os, 'O_BINARY', 0), # Windows only
  1269. getattr(os, 'O_NOINHERIT', 0), # Windows only
  1270. os.O_CREAT if writable else 0, # O_TRUNC only after locking
  1271. os.O_APPEND if 'a' in mode else 0,
  1272. os.O_EXCL if 'x' in mode else 0,
  1273. os.O_RDONLY if not writable else os.O_RDWR if readable else os.O_WRONLY,
  1274. ))
  1275. self.f = os.fdopen(os.open(filename, flags, 0o666), mode, encoding=encoding)
  1276. def __enter__(self):
  1277. exclusive = 'r' not in self.mode
  1278. try:
  1279. _lock_file(self.f, exclusive, self.block)
  1280. self.locked = True
  1281. except OSError:
  1282. self.f.close()
  1283. raise
  1284. if 'w' in self.mode:
  1285. try:
  1286. self.f.truncate()
  1287. except OSError as e:
  1288. if e.errno not in (
  1289. errno.ESPIPE, # Illegal seek - expected for FIFO
  1290. errno.EINVAL, # Invalid argument - expected for /dev/null
  1291. ):
  1292. raise
  1293. return self
  1294. def unlock(self):
  1295. if not self.locked:
  1296. return
  1297. try:
  1298. _unlock_file(self.f)
  1299. finally:
  1300. self.locked = False
  1301. def __exit__(self, *_):
  1302. try:
  1303. self.unlock()
  1304. finally:
  1305. self.f.close()
  1306. open = __enter__
  1307. close = __exit__
  1308. def __getattr__(self, attr):
  1309. return getattr(self.f, attr)
  1310. def __iter__(self):
  1311. return iter(self.f)
  1312. @functools.cache
  1313. def get_filesystem_encoding():
  1314. encoding = sys.getfilesystemencoding()
  1315. return encoding if encoding is not None else 'utf-8'
  1316. _WINDOWS_QUOTE_TRANS = str.maketrans({'"': R'\"'})
  1317. _CMD_QUOTE_TRANS = str.maketrans({
  1318. # Keep quotes balanced by replacing them with `""` instead of `\\"`
  1319. '"': '""',
  1320. # These require an env-variable `=` containing `"^\n\n"` (set in `utils.Popen`)
  1321. # `=` should be unique since variables containing `=` cannot be set using cmd
  1322. '\n': '%=%',
  1323. '\r': '%=%',
  1324. # Use zero length variable replacement so `%` doesn't get expanded
  1325. # `cd` is always set as long as extensions are enabled (`/E:ON` in `utils.Popen`)
  1326. '%': '%%cd:~,%',
  1327. })
  1328. def shell_quote(args, *, shell=False):
  1329. args = list(variadic(args))
  1330. if compat_os_name != 'nt':
  1331. return shlex.join(args)
  1332. trans = _CMD_QUOTE_TRANS if shell else _WINDOWS_QUOTE_TRANS
  1333. return ' '.join(
  1334. s if re.fullmatch(r'[\w#$*\-+./:?@\\]+', s, re.ASCII)
  1335. else re.sub(r'(\\+)("|$)', r'\1\1\2', s).translate(trans).join('""')
  1336. for s in args)
  1337. def smuggle_url(url, data):
  1338. """ Pass additional data in a URL for internal use. """
  1339. url, idata = unsmuggle_url(url, {})
  1340. data.update(idata)
  1341. sdata = urllib.parse.urlencode(
  1342. {'__youtubedl_smuggle': json.dumps(data)})
  1343. return url + '#' + sdata
  1344. def unsmuggle_url(smug_url, default=None):
  1345. if '#__youtubedl_smuggle' not in smug_url:
  1346. return smug_url, default
  1347. url, _, sdata = smug_url.rpartition('#')
  1348. jsond = urllib.parse.parse_qs(sdata)['__youtubedl_smuggle'][0]
  1349. data = json.loads(jsond)
  1350. return url, data
  1351. def format_decimal_suffix(num, fmt='%d%s', *, factor=1000):
  1352. """ Formats numbers with decimal sufixes like K, M, etc """
  1353. num, factor = float_or_none(num), float(factor)
  1354. if num is None or num < 0:
  1355. return None
  1356. POSSIBLE_SUFFIXES = 'kMGTPEZY'
  1357. exponent = 0 if num == 0 else min(int(math.log(num, factor)), len(POSSIBLE_SUFFIXES))
  1358. suffix = ['', *POSSIBLE_SUFFIXES][exponent]
  1359. if factor == 1024:
  1360. suffix = {'k': 'Ki', '': ''}.get(suffix, f'{suffix}i')
  1361. converted = num / (factor ** exponent)
  1362. return fmt % (converted, suffix)
  1363. def format_bytes(bytes):
  1364. return format_decimal_suffix(bytes, '%.2f%sB', factor=1024) or 'N/A'
  1365. def lookup_unit_table(unit_table, s, strict=False):
  1366. num_re = NUMBER_RE if strict else NUMBER_RE.replace(R'\.', '[,.]')
  1367. units_re = '|'.join(re.escape(u) for u in unit_table)
  1368. m = (re.fullmatch if strict else re.match)(
  1369. rf'(?P<num>{num_re})\s*(?P<unit>{units_re})\b', s)
  1370. if not m:
  1371. return None
  1372. num = float(m.group('num').replace(',', '.'))
  1373. mult = unit_table[m.group('unit')]
  1374. return round(num * mult)
  1375. def parse_bytes(s):
  1376. """Parse a string indicating a byte quantity into an integer"""
  1377. return lookup_unit_table(
  1378. {u: 1024**i for i, u in enumerate(['', *'KMGTPEZY'])},
  1379. s.upper(), strict=True)
  1380. def parse_filesize(s):
  1381. if s is None:
  1382. return None
  1383. # The lower-case forms are of course incorrect and unofficial,
  1384. # but we support those too
  1385. _UNIT_TABLE = {
  1386. 'B': 1,
  1387. 'b': 1,
  1388. 'bytes': 1,
  1389. 'KiB': 1024,
  1390. 'KB': 1000,
  1391. 'kB': 1024,
  1392. 'Kb': 1000,
  1393. 'kb': 1000,
  1394. 'kilobytes': 1000,
  1395. 'kibibytes': 1024,
  1396. 'MiB': 1024 ** 2,
  1397. 'MB': 1000 ** 2,
  1398. 'mB': 1024 ** 2,
  1399. 'Mb': 1000 ** 2,
  1400. 'mb': 1000 ** 2,
  1401. 'megabytes': 1000 ** 2,
  1402. 'mebibytes': 1024 ** 2,
  1403. 'GiB': 1024 ** 3,
  1404. 'GB': 1000 ** 3,
  1405. 'gB': 1024 ** 3,
  1406. 'Gb': 1000 ** 3,
  1407. 'gb': 1000 ** 3,
  1408. 'gigabytes': 1000 ** 3,
  1409. 'gibibytes': 1024 ** 3,
  1410. 'TiB': 1024 ** 4,
  1411. 'TB': 1000 ** 4,
  1412. 'tB': 1024 ** 4,
  1413. 'Tb': 1000 ** 4,
  1414. 'tb': 1000 ** 4,
  1415. 'terabytes': 1000 ** 4,
  1416. 'tebibytes': 1024 ** 4,
  1417. 'PiB': 1024 ** 5,
  1418. 'PB': 1000 ** 5,
  1419. 'pB': 1024 ** 5,
  1420. 'Pb': 1000 ** 5,
  1421. 'pb': 1000 ** 5,
  1422. 'petabytes': 1000 ** 5,
  1423. 'pebibytes': 1024 ** 5,
  1424. 'EiB': 1024 ** 6,
  1425. 'EB': 1000 ** 6,
  1426. 'eB': 1024 ** 6,
  1427. 'Eb': 1000 ** 6,
  1428. 'eb': 1000 ** 6,
  1429. 'exabytes': 1000 ** 6,
  1430. 'exbibytes': 1024 ** 6,
  1431. 'ZiB': 1024 ** 7,
  1432. 'ZB': 1000 ** 7,
  1433. 'zB': 1024 ** 7,
  1434. 'Zb': 1000 ** 7,
  1435. 'zb': 1000 ** 7,
  1436. 'zettabytes': 1000 ** 7,
  1437. 'zebibytes': 1024 ** 7,
  1438. 'YiB': 1024 ** 8,
  1439. 'YB': 1000 ** 8,
  1440. 'yB': 1024 ** 8,
  1441. 'Yb': 1000 ** 8,
  1442. 'yb': 1000 ** 8,
  1443. 'yottabytes': 1000 ** 8,
  1444. 'yobibytes': 1024 ** 8,
  1445. }
  1446. return lookup_unit_table(_UNIT_TABLE, s)
  1447. def parse_count(s):
  1448. if s is None:
  1449. return None
  1450. s = re.sub(r'^[^\d]+\s', '', s).strip()
  1451. if re.match(r'^[\d,.]+$', s):
  1452. return str_to_int(s)
  1453. _UNIT_TABLE = {
  1454. 'k': 1000,
  1455. 'K': 1000,
  1456. 'm': 1000 ** 2,
  1457. 'M': 1000 ** 2,
  1458. 'kk': 1000 ** 2,
  1459. 'KK': 1000 ** 2,
  1460. 'b': 1000 ** 3,
  1461. 'B': 1000 ** 3,
  1462. }
  1463. ret = lookup_unit_table(_UNIT_TABLE, s)
  1464. if ret is not None:
  1465. return ret
  1466. mobj = re.match(r'([\d,.]+)(?:$|\s)', s)
  1467. if mobj:
  1468. return str_to_int(mobj.group(1))
  1469. def parse_resolution(s, *, lenient=False):
  1470. if s is None:
  1471. return {}
  1472. if lenient:
  1473. mobj = re.search(r'(?P<w>\d+)\s*[xX×,]\s*(?P<h>\d+)', s)
  1474. else:
  1475. mobj = re.search(r'(?<![a-zA-Z0-9])(?P<w>\d+)\s*[xX×,]\s*(?P<h>\d+)(?![a-zA-Z0-9])', s)
  1476. if mobj:
  1477. return {
  1478. 'width': int(mobj.group('w')),
  1479. 'height': int(mobj.group('h')),
  1480. }
  1481. mobj = re.search(r'(?<![a-zA-Z0-9])(\d+)[pPiI](?![a-zA-Z0-9])', s)
  1482. if mobj:
  1483. return {'height': int(mobj.group(1))}
  1484. mobj = re.search(r'\b([48])[kK]\b', s)
  1485. if mobj:
  1486. return {'height': int(mobj.group(1)) * 540}
  1487. return {}
  1488. def parse_bitrate(s):
  1489. if not isinstance(s, str):
  1490. return
  1491. mobj = re.search(r'\b(\d+)\s*kbps', s)
  1492. if mobj:
  1493. return int(mobj.group(1))
  1494. def month_by_name(name, lang='en'):
  1495. """ Return the number of a month by (locale-independently) English name """
  1496. month_names = MONTH_NAMES.get(lang, MONTH_NAMES['en'])
  1497. try:
  1498. return month_names.index(name) + 1
  1499. except ValueError:
  1500. return None
  1501. def month_by_abbreviation(abbrev):
  1502. """ Return the number of a month by (locale-independently) English
  1503. abbreviations """
  1504. try:
  1505. return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
  1506. except ValueError:
  1507. return None
  1508. def fix_xml_ampersands(xml_str):
  1509. """Replace all the '&' by '&amp;' in XML"""
  1510. return re.sub(
  1511. r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
  1512. '&amp;',
  1513. xml_str)
  1514. def setproctitle(title):
  1515. assert isinstance(title, str)
  1516. # Workaround for https://github.com/yt-dlp/yt-dlp/issues/4541
  1517. try:
  1518. import ctypes
  1519. except ImportError:
  1520. return
  1521. try:
  1522. libc = ctypes.cdll.LoadLibrary('libc.so.6')
  1523. except OSError:
  1524. return
  1525. except TypeError:
  1526. # LoadLibrary in Windows Python 2.7.13 only expects
  1527. # a bytestring, but since unicode_literals turns
  1528. # every string into a unicode string, it fails.
  1529. return
  1530. title_bytes = title.encode()
  1531. buf = ctypes.create_string_buffer(len(title_bytes))
  1532. buf.value = title_bytes
  1533. try:
  1534. # PR_SET_NAME = 15 Ref: /usr/include/linux/prctl.h
  1535. libc.prctl(15, buf, 0, 0, 0)
  1536. except AttributeError:
  1537. return # Strange libc, just skip this
  1538. def remove_start(s, start):
  1539. return s[len(start):] if s is not None and s.startswith(start) else s
  1540. def remove_end(s, end):
  1541. return s[:-len(end)] if s is not None and s.endswith(end) else s
  1542. def remove_quotes(s):
  1543. if s is None or len(s) < 2:
  1544. return s
  1545. for quote in ('"', "'"):
  1546. if s[0] == quote and s[-1] == quote:
  1547. return s[1:-1]
  1548. return s
  1549. def get_domain(url):
  1550. """
  1551. This implementation is inconsistent, but is kept for compatibility.
  1552. Use this only for "webpage_url_domain"
  1553. """
  1554. return remove_start(urllib.parse.urlparse(url).netloc, 'www.') or None
  1555. def url_basename(url):
  1556. path = urllib.parse.urlparse(url).path
  1557. return path.strip('/').split('/')[-1]
  1558. def base_url(url):
  1559. return re.match(r'https?://[^?#]+/', url).group()
  1560. def urljoin(base, path):
  1561. if isinstance(path, bytes):
  1562. path = path.decode()
  1563. if not isinstance(path, str) or not path:
  1564. return None
  1565. if re.match(r'(?:[a-zA-Z][a-zA-Z0-9+-.]*:)?//', path):
  1566. return path
  1567. if isinstance(base, bytes):
  1568. base = base.decode()
  1569. if not isinstance(base, str) or not re.match(
  1570. r'^(?:https?:)?//', base):
  1571. return None
  1572. return urllib.parse.urljoin(base, path)
  1573. def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
  1574. if get_attr and v is not None:
  1575. v = getattr(v, get_attr, None)
  1576. try:
  1577. return int(v) * invscale // scale
  1578. except (ValueError, TypeError, OverflowError):
  1579. return default
  1580. def str_or_none(v, default=None):
  1581. return default if v is None else str(v)
  1582. def str_to_int(int_str):
  1583. """ A more relaxed version of int_or_none """
  1584. if isinstance(int_str, int):
  1585. return int_str
  1586. elif isinstance(int_str, str):
  1587. int_str = re.sub(r'[,\.\+]', '', int_str)
  1588. return int_or_none(int_str)
  1589. def float_or_none(v, scale=1, invscale=1, default=None):
  1590. if v is None:
  1591. return default
  1592. try:
  1593. return float(v) * invscale / scale
  1594. except (ValueError, TypeError):
  1595. return default
  1596. def bool_or_none(v, default=None):
  1597. return v if isinstance(v, bool) else default
  1598. def strip_or_none(v, default=None):
  1599. return v.strip() if isinstance(v, str) else default
  1600. def url_or_none(url):
  1601. if not url or not isinstance(url, str):
  1602. return None
  1603. url = url.strip()
  1604. return url if re.match(r'(?:(?:https?|rt(?:m(?:pt?[es]?|fp)|sp[su]?)|mms|ftps?):)?//', url) else None
  1605. def strftime_or_none(timestamp, date_format='%Y%m%d', default=None):
  1606. datetime_object = None
  1607. try:
  1608. if isinstance(timestamp, (int, float)): # unix timestamp
  1609. # Using naive datetime here can break timestamp() in Windows
  1610. # Ref: https://github.com/yt-dlp/yt-dlp/issues/5185, https://github.com/python/cpython/issues/94414
  1611. # Also, dt.datetime.fromtimestamp breaks for negative timestamps
  1612. # Ref: https://github.com/yt-dlp/yt-dlp/issues/6706#issuecomment-1496842642
  1613. datetime_object = (dt.datetime.fromtimestamp(0, dt.timezone.utc)
  1614. + dt.timedelta(seconds=timestamp))
  1615. elif isinstance(timestamp, str): # assume YYYYMMDD
  1616. datetime_object = dt.datetime.strptime(timestamp, '%Y%m%d')
  1617. date_format = re.sub( # Support %s on windows
  1618. r'(?<!%)(%%)*%s', rf'\g<1>{int(datetime_object.timestamp())}', date_format)
  1619. return datetime_object.strftime(date_format)
  1620. except (ValueError, TypeError, AttributeError):
  1621. return default
  1622. def parse_duration(s):
  1623. if not isinstance(s, str):
  1624. return None
  1625. s = s.strip()
  1626. if not s:
  1627. return None
  1628. days, hours, mins, secs, ms = [None] * 5
  1629. m = re.match(r'''(?x)
  1630. (?P<before_secs>
  1631. (?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?
  1632. (?P<secs>(?(before_secs)[0-9]{1,2}|[0-9]+))
  1633. (?P<ms>[.:][0-9]+)?Z?$
  1634. ''', s)
  1635. if m:
  1636. days, hours, mins, secs, ms = m.group('days', 'hours', 'mins', 'secs', 'ms')
  1637. else:
  1638. m = re.match(
  1639. r'''(?ix)(?:P?
  1640. (?:
  1641. [0-9]+\s*y(?:ears?)?,?\s*
  1642. )?
  1643. (?:
  1644. [0-9]+\s*m(?:onths?)?,?\s*
  1645. )?
  1646. (?:
  1647. [0-9]+\s*w(?:eeks?)?,?\s*
  1648. )?
  1649. (?:
  1650. (?P<days>[0-9]+)\s*d(?:ays?)?,?\s*
  1651. )?
  1652. T)?
  1653. (?:
  1654. (?P<hours>[0-9]+)\s*h(?:(?:ou)?rs?)?,?\s*
  1655. )?
  1656. (?:
  1657. (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?,?\s*
  1658. )?
  1659. (?:
  1660. (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
  1661. )?Z?$''', s)
  1662. if m:
  1663. days, hours, mins, secs, ms = m.groups()
  1664. else:
  1665. m = re.match(r'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)Z?$', s)
  1666. if m:
  1667. hours, mins = m.groups()
  1668. else:
  1669. return None
  1670. if ms:
  1671. ms = ms.replace(':', '.')
  1672. return sum(float(part or 0) * mult for part, mult in (
  1673. (days, 86400), (hours, 3600), (mins, 60), (secs, 1), (ms, 1)))
  1674. def _change_extension(prepend, filename, ext, expected_real_ext=None):
  1675. name, real_ext = os.path.splitext(filename)
  1676. if not expected_real_ext or real_ext[1:] == expected_real_ext:
  1677. filename = name
  1678. if prepend and real_ext:
  1679. _UnsafeExtensionError.sanitize_extension(ext, prepend=True)
  1680. return f'{filename}.{ext}{real_ext}'
  1681. return f'{filename}.{_UnsafeExtensionError.sanitize_extension(ext)}'
  1682. prepend_extension = functools.partial(_change_extension, True)
  1683. replace_extension = functools.partial(_change_extension, False)
  1684. def check_executable(exe, args=[]):
  1685. """ Checks if the given binary is installed somewhere in PATH, and returns its name.
  1686. args can be a list of arguments for a short output (like -version) """
  1687. try:
  1688. Popen.run([exe, *args], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  1689. except OSError:
  1690. return False
  1691. return exe
  1692. def _get_exe_version_output(exe, args):
  1693. try:
  1694. # STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers
  1695. # SIGTTOU if yt-dlp is run in the background.
  1696. # See https://github.com/ytdl-org/youtube-dl/issues/955#issuecomment-209789656
  1697. stdout, _, ret = Popen.run([encodeArgument(exe), *args], text=True,
  1698. stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  1699. if ret:
  1700. return None
  1701. except OSError:
  1702. return False
  1703. return stdout
  1704. def detect_exe_version(output, version_re=None, unrecognized='present'):
  1705. assert isinstance(output, str)
  1706. if version_re is None:
  1707. version_re = r'version\s+([-0-9._a-zA-Z]+)'
  1708. m = re.search(version_re, output)
  1709. if m:
  1710. return m.group(1)
  1711. else:
  1712. return unrecognized
  1713. def get_exe_version(exe, args=['--version'],
  1714. version_re=None, unrecognized=('present', 'broken')):
  1715. """ Returns the version of the specified executable,
  1716. or False if the executable is not present """
  1717. unrecognized = variadic(unrecognized)
  1718. assert len(unrecognized) in (1, 2)
  1719. out = _get_exe_version_output(exe, args)
  1720. if out is None:
  1721. return unrecognized[-1]
  1722. return out and detect_exe_version(out, version_re, unrecognized[0])
  1723. def frange(start=0, stop=None, step=1):
  1724. """Float range"""
  1725. if stop is None:
  1726. start, stop = 0, start
  1727. sign = [-1, 1][step > 0] if step else 0
  1728. while sign * start < sign * stop:
  1729. yield start
  1730. start += step
  1731. class LazyList(collections.abc.Sequence):
  1732. """Lazy immutable list from an iterable
  1733. Note that slices of a LazyList are lists and not LazyList"""
  1734. class IndexError(IndexError): # noqa: A001
  1735. pass
  1736. def __init__(self, iterable, *, reverse=False, _cache=None):
  1737. self._iterable = iter(iterable)
  1738. self._cache = [] if _cache is None else _cache
  1739. self._reversed = reverse
  1740. def __iter__(self):
  1741. if self._reversed:
  1742. # We need to consume the entire iterable to iterate in reverse
  1743. yield from self.exhaust()
  1744. return
  1745. yield from self._cache
  1746. for item in self._iterable:
  1747. self._cache.append(item)
  1748. yield item
  1749. def _exhaust(self):
  1750. self._cache.extend(self._iterable)
  1751. self._iterable = [] # Discard the emptied iterable to make it pickle-able
  1752. return self._cache
  1753. def exhaust(self):
  1754. """Evaluate the entire iterable"""
  1755. return self._exhaust()[::-1 if self._reversed else 1]
  1756. @staticmethod
  1757. def _reverse_index(x):
  1758. return None if x is None else ~x
  1759. def __getitem__(self, idx):
  1760. if isinstance(idx, slice):
  1761. if self._reversed:
  1762. idx = slice(self._reverse_index(idx.start), self._reverse_index(idx.stop), -(idx.step or 1))
  1763. start, stop, step = idx.start, idx.stop, idx.step or 1
  1764. elif isinstance(idx, int):
  1765. if self._reversed:
  1766. idx = self._reverse_index(idx)
  1767. start, stop, step = idx, idx, 0
  1768. else:
  1769. raise TypeError('indices must be integers or slices')
  1770. if ((start or 0) < 0 or (stop or 0) < 0
  1771. or (start is None and step < 0)
  1772. or (stop is None and step > 0)):
  1773. # We need to consume the entire iterable to be able to slice from the end
  1774. # Obviously, never use this with infinite iterables
  1775. self._exhaust()
  1776. try:
  1777. return self._cache[idx]
  1778. except IndexError as e:
  1779. raise self.IndexError(e) from e
  1780. n = max(start or 0, stop or 0) - len(self._cache) + 1
  1781. if n > 0:
  1782. self._cache.extend(itertools.islice(self._iterable, n))
  1783. try:
  1784. return self._cache[idx]
  1785. except IndexError as e:
  1786. raise self.IndexError(e) from e
  1787. def __bool__(self):
  1788. try:
  1789. self[-1] if self._reversed else self[0]
  1790. except self.IndexError:
  1791. return False
  1792. return True
  1793. def __len__(self):
  1794. self._exhaust()
  1795. return len(self._cache)
  1796. def __reversed__(self):
  1797. return type(self)(self._iterable, reverse=not self._reversed, _cache=self._cache)
  1798. def __copy__(self):
  1799. return type(self)(self._iterable, reverse=self._reversed, _cache=self._cache)
  1800. def __repr__(self):
  1801. # repr and str should mimic a list. So we exhaust the iterable
  1802. return repr(self.exhaust())
  1803. def __str__(self):
  1804. return repr(self.exhaust())
  1805. class PagedList:
  1806. class IndexError(IndexError): # noqa: A001
  1807. pass
  1808. def __len__(self):
  1809. # This is only useful for tests
  1810. return len(self.getslice())
  1811. def __init__(self, pagefunc, pagesize, use_cache=True):
  1812. self._pagefunc = pagefunc
  1813. self._pagesize = pagesize
  1814. self._pagecount = float('inf')
  1815. self._use_cache = use_cache
  1816. self._cache = {}
  1817. def getpage(self, pagenum):
  1818. page_results = self._cache.get(pagenum)
  1819. if page_results is None:
  1820. page_results = [] if pagenum > self._pagecount else list(self._pagefunc(pagenum))
  1821. if self._use_cache:
  1822. self._cache[pagenum] = page_results
  1823. return page_results
  1824. def getslice(self, start=0, end=None):
  1825. return list(self._getslice(start, end))
  1826. def _getslice(self, start, end):
  1827. raise NotImplementedError('This method must be implemented by subclasses')
  1828. def __getitem__(self, idx):
  1829. assert self._use_cache, 'Indexing PagedList requires cache'
  1830. if not isinstance(idx, int) or idx < 0:
  1831. raise TypeError('indices must be non-negative integers')
  1832. entries = self.getslice(idx, idx + 1)
  1833. if not entries:
  1834. raise self.IndexError
  1835. return entries[0]
  1836. def __bool__(self):
  1837. return bool(self.getslice(0, 1))
  1838. class OnDemandPagedList(PagedList):
  1839. """Download pages until a page with less than maximum results"""
  1840. def _getslice(self, start, end):
  1841. for pagenum in itertools.count(start // self._pagesize):
  1842. firstid = pagenum * self._pagesize
  1843. nextfirstid = pagenum * self._pagesize + self._pagesize
  1844. if start >= nextfirstid:
  1845. continue
  1846. startv = (
  1847. start % self._pagesize
  1848. if firstid <= start < nextfirstid
  1849. else 0)
  1850. endv = (
  1851. ((end - 1) % self._pagesize) + 1
  1852. if (end is not None and firstid <= end <= nextfirstid)
  1853. else None)
  1854. try:
  1855. page_results = self.getpage(pagenum)
  1856. except Exception:
  1857. self._pagecount = pagenum - 1
  1858. raise
  1859. if startv != 0 or endv is not None:
  1860. page_results = page_results[startv:endv]
  1861. yield from page_results
  1862. # A little optimization - if current page is not "full", ie. does
  1863. # not contain page_size videos then we can assume that this page
  1864. # is the last one - there are no more ids on further pages -
  1865. # i.e. no need to query again.
  1866. if len(page_results) + startv < self._pagesize:
  1867. break
  1868. # If we got the whole page, but the next page is not interesting,
  1869. # break out early as well
  1870. if end == nextfirstid:
  1871. break
  1872. class InAdvancePagedList(PagedList):
  1873. """PagedList with total number of pages known in advance"""
  1874. def __init__(self, pagefunc, pagecount, pagesize):
  1875. PagedList.__init__(self, pagefunc, pagesize, True)
  1876. self._pagecount = pagecount
  1877. def _getslice(self, start, end):
  1878. start_page = start // self._pagesize
  1879. end_page = self._pagecount if end is None else min(self._pagecount, end // self._pagesize + 1)
  1880. skip_elems = start - start_page * self._pagesize
  1881. only_more = None if end is None else end - start
  1882. for pagenum in range(start_page, end_page):
  1883. page_results = self.getpage(pagenum)
  1884. if skip_elems:
  1885. page_results = page_results[skip_elems:]
  1886. skip_elems = None
  1887. if only_more is not None:
  1888. if len(page_results) < only_more:
  1889. only_more -= len(page_results)
  1890. else:
  1891. yield from page_results[:only_more]
  1892. break
  1893. yield from page_results
  1894. class PlaylistEntries:
  1895. MissingEntry = object()
  1896. is_exhausted = False
  1897. def __init__(self, ydl, info_dict):
  1898. self.ydl = ydl
  1899. # _entries must be assigned now since infodict can change during iteration
  1900. entries = info_dict.get('entries')
  1901. if entries is None:
  1902. raise EntryNotInPlaylist('There are no entries')
  1903. elif isinstance(entries, list):
  1904. self.is_exhausted = True
  1905. requested_entries = info_dict.get('requested_entries')
  1906. self.is_incomplete = requested_entries is not None
  1907. if self.is_incomplete:
  1908. assert self.is_exhausted
  1909. self._entries = [self.MissingEntry] * max(requested_entries or [0])
  1910. for i, entry in zip(requested_entries, entries):
  1911. self._entries[i - 1] = entry
  1912. elif isinstance(entries, (list, PagedList, LazyList)):
  1913. self._entries = entries
  1914. else:
  1915. self._entries = LazyList(entries)
  1916. PLAYLIST_ITEMS_RE = re.compile(r'''(?x)
  1917. (?P<start>[+-]?\d+)?
  1918. (?P<range>[:-]
  1919. (?P<end>[+-]?\d+|inf(?:inite)?)?
  1920. (?::(?P<step>[+-]?\d+))?
  1921. )?''')
  1922. @classmethod
  1923. def parse_playlist_items(cls, string):
  1924. for segment in string.split(','):
  1925. if not segment:
  1926. raise ValueError('There is two or more consecutive commas')
  1927. mobj = cls.PLAYLIST_ITEMS_RE.fullmatch(segment)
  1928. if not mobj:
  1929. raise ValueError(f'{segment!r} is not a valid specification')
  1930. start, end, step, has_range = mobj.group('start', 'end', 'step', 'range')
  1931. if int_or_none(step) == 0:
  1932. raise ValueError(f'Step in {segment!r} cannot be zero')
  1933. yield slice(int_or_none(start), float_or_none(end), int_or_none(step)) if has_range else int(start)
  1934. def get_requested_items(self):
  1935. playlist_items = self.ydl.params.get('playlist_items')
  1936. playlist_start = self.ydl.params.get('playliststart', 1)
  1937. playlist_end = self.ydl.params.get('playlistend')
  1938. # For backwards compatibility, interpret -1 as whole list
  1939. if playlist_end in (-1, None):
  1940. playlist_end = ''
  1941. if not playlist_items:
  1942. playlist_items = f'{playlist_start}:{playlist_end}'
  1943. elif playlist_start != 1 or playlist_end:
  1944. self.ydl.report_warning('Ignoring playliststart and playlistend because playlistitems was given', only_once=True)
  1945. for index in self.parse_playlist_items(playlist_items):
  1946. for i, entry in self[index]:
  1947. yield i, entry
  1948. if not entry:
  1949. continue
  1950. try:
  1951. # The item may have just been added to archive. Don't break due to it
  1952. if not self.ydl.params.get('lazy_playlist'):
  1953. # TODO: Add auto-generated fields
  1954. self.ydl._match_entry(entry, incomplete=True, silent=True)
  1955. except (ExistingVideoReached, RejectedVideoReached):
  1956. return
  1957. def get_full_count(self):
  1958. if self.is_exhausted and not self.is_incomplete:
  1959. return len(self)
  1960. elif isinstance(self._entries, InAdvancePagedList):
  1961. if self._entries._pagesize == 1:
  1962. return self._entries._pagecount
  1963. @functools.cached_property
  1964. def _getter(self):
  1965. if isinstance(self._entries, list):
  1966. def get_entry(i):
  1967. try:
  1968. entry = self._entries[i]
  1969. except IndexError:
  1970. entry = self.MissingEntry
  1971. if not self.is_incomplete:
  1972. raise self.IndexError
  1973. if entry is self.MissingEntry:
  1974. raise EntryNotInPlaylist(f'Entry {i + 1} cannot be found')
  1975. return entry
  1976. else:
  1977. def get_entry(i):
  1978. try:
  1979. return type(self.ydl)._handle_extraction_exceptions(lambda _, i: self._entries[i])(self.ydl, i)
  1980. except (LazyList.IndexError, PagedList.IndexError):
  1981. raise self.IndexError
  1982. return get_entry
  1983. def __getitem__(self, idx):
  1984. if isinstance(idx, int):
  1985. idx = slice(idx, idx)
  1986. # NB: PlaylistEntries[1:10] => (0, 1, ... 9)
  1987. step = 1 if idx.step is None else idx.step
  1988. if idx.start is None:
  1989. start = 0 if step > 0 else len(self) - 1
  1990. else:
  1991. start = idx.start - 1 if idx.start >= 0 else len(self) + idx.start
  1992. # NB: Do not call len(self) when idx == [:]
  1993. if idx.stop is None:
  1994. stop = 0 if step < 0 else float('inf')
  1995. else:
  1996. stop = idx.stop - 1 if idx.stop >= 0 else len(self) + idx.stop
  1997. stop += [-1, 1][step > 0]
  1998. for i in frange(start, stop, step):
  1999. if i < 0:
  2000. continue
  2001. try:
  2002. entry = self._getter(i)
  2003. except self.IndexError:
  2004. self.is_exhausted = True
  2005. if step > 0:
  2006. break
  2007. continue
  2008. yield i + 1, entry
  2009. def __len__(self):
  2010. return len(tuple(self[:]))
  2011. class IndexError(IndexError): # noqa: A001
  2012. pass
  2013. def uppercase_escape(s):
  2014. unicode_escape = codecs.getdecoder('unicode_escape')
  2015. return re.sub(
  2016. r'\\U[0-9a-fA-F]{8}',
  2017. lambda m: unicode_escape(m.group(0))[0],
  2018. s)
  2019. def lowercase_escape(s):
  2020. unicode_escape = codecs.getdecoder('unicode_escape')
  2021. return re.sub(
  2022. r'\\u[0-9a-fA-F]{4}',
  2023. lambda m: unicode_escape(m.group(0))[0],
  2024. s)
  2025. def parse_qs(url, **kwargs):
  2026. return urllib.parse.parse_qs(urllib.parse.urlparse(url).query, **kwargs)
  2027. def read_batch_urls(batch_fd):
  2028. def fixup(url):
  2029. if not isinstance(url, str):
  2030. url = url.decode('utf-8', 'replace')
  2031. BOM_UTF8 = ('\xef\xbb\xbf', '\ufeff')
  2032. for bom in BOM_UTF8:
  2033. if url.startswith(bom):
  2034. url = url[len(bom):]
  2035. url = url.lstrip()
  2036. if not url or url.startswith(('#', ';', ']')):
  2037. return False
  2038. # "#" cannot be stripped out since it is part of the URI
  2039. # However, it can be safely stripped out if following a whitespace
  2040. return re.split(r'\s#', url, maxsplit=1)[0].rstrip()
  2041. with contextlib.closing(batch_fd) as fd:
  2042. return [url for url in map(fixup, fd) if url]
  2043. def urlencode_postdata(*args, **kargs):
  2044. return urllib.parse.urlencode(*args, **kargs).encode('ascii')
  2045. def update_url(url, *, query_update=None, **kwargs):
  2046. """Replace URL components specified by kwargs
  2047. @param url str or parse url tuple
  2048. @param query_update update query
  2049. @returns str
  2050. """
  2051. if isinstance(url, str):
  2052. if not kwargs and not query_update:
  2053. return url
  2054. else:
  2055. url = urllib.parse.urlparse(url)
  2056. if query_update:
  2057. assert 'query' not in kwargs, 'query_update and query cannot be specified at the same time'
  2058. kwargs['query'] = urllib.parse.urlencode({
  2059. **urllib.parse.parse_qs(url.query),
  2060. **query_update,
  2061. }, True)
  2062. return urllib.parse.urlunparse(url._replace(**kwargs))
  2063. def update_url_query(url, query):
  2064. return update_url(url, query_update=query)
  2065. def _multipart_encode_impl(data, boundary):
  2066. content_type = f'multipart/form-data; boundary={boundary}'
  2067. out = b''
  2068. for k, v in data.items():
  2069. out += b'--' + boundary.encode('ascii') + b'\r\n'
  2070. if isinstance(k, str):
  2071. k = k.encode()
  2072. if isinstance(v, str):
  2073. v = v.encode()
  2074. # RFC 2047 requires non-ASCII field names to be encoded, while RFC 7578
  2075. # suggests sending UTF-8 directly. Firefox sends UTF-8, too
  2076. content = b'Content-Disposition: form-data; name="' + k + b'"\r\n\r\n' + v + b'\r\n'
  2077. if boundary.encode('ascii') in content:
  2078. raise ValueError('Boundary overlaps with data')
  2079. out += content
  2080. out += b'--' + boundary.encode('ascii') + b'--\r\n'
  2081. return out, content_type
  2082. def multipart_encode(data, boundary=None):
  2083. """
  2084. Encode a dict to RFC 7578-compliant form-data
  2085. data:
  2086. A dict where keys and values can be either Unicode or bytes-like
  2087. objects.
  2088. boundary:
  2089. If specified a Unicode object, it's used as the boundary. Otherwise
  2090. a random boundary is generated.
  2091. Reference: https://tools.ietf.org/html/rfc7578
  2092. """
  2093. has_specified_boundary = boundary is not None
  2094. while True:
  2095. if boundary is None:
  2096. boundary = '---------------' + str(random.randrange(0x0fffffff, 0xffffffff))
  2097. try:
  2098. out, content_type = _multipart_encode_impl(data, boundary)
  2099. break
  2100. except ValueError:
  2101. if has_specified_boundary:
  2102. raise
  2103. boundary = None
  2104. return out, content_type
  2105. def is_iterable_like(x, allowed_types=collections.abc.Iterable, blocked_types=NO_DEFAULT):
  2106. if blocked_types is NO_DEFAULT:
  2107. blocked_types = (str, bytes, collections.abc.Mapping)
  2108. return isinstance(x, allowed_types) and not isinstance(x, blocked_types)
  2109. def variadic(x, allowed_types=NO_DEFAULT):
  2110. if not isinstance(allowed_types, (tuple, type)):
  2111. deprecation_warning('allowed_types should be a tuple or a type')
  2112. allowed_types = tuple(allowed_types)
  2113. return x if is_iterable_like(x, blocked_types=allowed_types) else (x, )
  2114. def try_call(*funcs, expected_type=None, args=[], kwargs={}):
  2115. for f in funcs:
  2116. try:
  2117. val = f(*args, **kwargs)
  2118. except (AttributeError, KeyError, TypeError, IndexError, ValueError, ZeroDivisionError):
  2119. pass
  2120. else:
  2121. if expected_type is None or isinstance(val, expected_type):
  2122. return val
  2123. def try_get(src, getter, expected_type=None):
  2124. return try_call(*variadic(getter), args=(src,), expected_type=expected_type)
  2125. def filter_dict(dct, cndn=lambda _, v: v is not None):
  2126. return {k: v for k, v in dct.items() if cndn(k, v)}
  2127. def merge_dicts(*dicts):
  2128. merged = {}
  2129. for a_dict in dicts:
  2130. for k, v in a_dict.items():
  2131. if (v is not None and k not in merged
  2132. or isinstance(v, str) and merged[k] == ''):
  2133. merged[k] = v
  2134. return merged
  2135. def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
  2136. return string if isinstance(string, str) else str(string, encoding, errors)
  2137. US_RATINGS = {
  2138. 'G': 0,
  2139. 'PG': 10,
  2140. 'PG-13': 13,
  2141. 'R': 16,
  2142. 'NC': 18,
  2143. }
  2144. TV_PARENTAL_GUIDELINES = {
  2145. 'TV-Y': 0,
  2146. 'TV-Y7': 7,
  2147. 'TV-G': 0,
  2148. 'TV-PG': 0,
  2149. 'TV-14': 14,
  2150. 'TV-MA': 17,
  2151. }
  2152. def parse_age_limit(s):
  2153. # isinstance(False, int) is True. So type() must be used instead
  2154. if type(s) is int: # noqa: E721
  2155. return s if 0 <= s <= 21 else None
  2156. elif not isinstance(s, str):
  2157. return None
  2158. m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
  2159. if m:
  2160. return int(m.group('age'))
  2161. s = s.upper()
  2162. if s in US_RATINGS:
  2163. return US_RATINGS[s]
  2164. m = re.match(r'^TV[_-]?({})$'.format('|'.join(k[3:] for k in TV_PARENTAL_GUIDELINES)), s)
  2165. if m:
  2166. return TV_PARENTAL_GUIDELINES['TV-' + m.group(1)]
  2167. return None
  2168. def strip_jsonp(code):
  2169. return re.sub(
  2170. r'''(?sx)^
  2171. (?:window\.)?(?P<func_name>[a-zA-Z0-9_.$]*)
  2172. (?:\s*&&\s*(?P=func_name))?
  2173. \s*\(\s*(?P<callback_data>.*)\);?
  2174. \s*?(?://[^\n]*)*$''',
  2175. r'\g<callback_data>', code)
  2176. def js_to_json(code, vars={}, *, strict=False):
  2177. # vars is a dict of var, val pairs to substitute
  2178. STRING_QUOTES = '\'"`'
  2179. STRING_RE = '|'.join(rf'{q}(?:\\.|[^\\{q}])*{q}' for q in STRING_QUOTES)
  2180. COMMENT_RE = r'/\*(?:(?!\*/).)*?\*/|//[^\n]*\n'
  2181. SKIP_RE = fr'\s*(?:{COMMENT_RE})?\s*'
  2182. INTEGER_TABLE = (
  2183. (fr'(?s)^(0[xX][0-9a-fA-F]+){SKIP_RE}:?$', 16),
  2184. (fr'(?s)^(0+[0-7]+){SKIP_RE}:?$', 8),
  2185. )
  2186. def process_escape(match):
  2187. JSON_PASSTHROUGH_ESCAPES = R'"\bfnrtu'
  2188. escape = match.group(1) or match.group(2)
  2189. return (Rf'\{escape}' if escape in JSON_PASSTHROUGH_ESCAPES
  2190. else R'\u00' if escape == 'x'
  2191. else '' if escape == '\n'
  2192. else escape)
  2193. def template_substitute(match):
  2194. evaluated = js_to_json(match.group(1), vars, strict=strict)
  2195. if evaluated[0] == '"':
  2196. return json.loads(evaluated)
  2197. return evaluated
  2198. def fix_kv(m):
  2199. v = m.group(0)
  2200. if v in ('true', 'false', 'null'):
  2201. return v
  2202. elif v in ('undefined', 'void 0'):
  2203. return 'null'
  2204. elif v.startswith(('/*', '//', '!')) or v == ',':
  2205. return ''
  2206. if v[0] in STRING_QUOTES:
  2207. v = re.sub(r'(?s)\${([^}]+)}', template_substitute, v[1:-1]) if v[0] == '`' else v[1:-1]
  2208. escaped = re.sub(r'(?s)(")|\\(.)', process_escape, v)
  2209. return f'"{escaped}"'
  2210. for regex, base in INTEGER_TABLE:
  2211. im = re.match(regex, v)
  2212. if im:
  2213. i = int(im.group(1), base)
  2214. return f'"{i}":' if v.endswith(':') else str(i)
  2215. if v in vars:
  2216. try:
  2217. if not strict:
  2218. json.loads(vars[v])
  2219. except json.JSONDecodeError:
  2220. return json.dumps(vars[v])
  2221. else:
  2222. return vars[v]
  2223. if not strict:
  2224. return f'"{v}"'
  2225. raise ValueError(f'Unknown value: {v}')
  2226. def create_map(mobj):
  2227. return json.dumps(dict(json.loads(js_to_json(mobj.group(1) or '[]', vars=vars))))
  2228. code = re.sub(r'(?:new\s+)?Array\((.*?)\)', r'[\g<1>]', code)
  2229. code = re.sub(r'new Map\((\[.*?\])?\)', create_map, code)
  2230. if not strict:
  2231. code = re.sub(rf'new Date\(({STRING_RE})\)', r'\g<1>', code)
  2232. code = re.sub(r'new \w+\((.*?)\)', lambda m: json.dumps(m.group(0)), code)
  2233. code = re.sub(r'parseInt\([^\d]+(\d+)[^\d]+\)', r'\1', code)
  2234. code = re.sub(r'\(function\([^)]*\)\s*\{[^}]*\}\s*\)\s*\(\s*(["\'][^)]*["\'])\s*\)', r'\1', code)
  2235. return re.sub(rf'''(?sx)
  2236. {STRING_RE}|
  2237. {COMMENT_RE}|,(?={SKIP_RE}[\]}}])|
  2238. void\s0|(?:(?<![0-9])[eE]|[a-df-zA-DF-Z_$])[.a-zA-Z_$0-9]*|
  2239. \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:{SKIP_RE}:)?|
  2240. [0-9]+(?={SKIP_RE}:)|
  2241. !+
  2242. ''', fix_kv, code)
  2243. def qualities(quality_ids):
  2244. """ Get a numeric quality value out of a list of possible values """
  2245. def q(qid):
  2246. try:
  2247. return quality_ids.index(qid)
  2248. except ValueError:
  2249. return -1
  2250. return q
  2251. POSTPROCESS_WHEN = ('pre_process', 'after_filter', 'video', 'before_dl', 'post_process', 'after_move', 'after_video', 'playlist')
  2252. DEFAULT_OUTTMPL = {
  2253. 'default': '%(title)s [%(id)s].%(ext)s',
  2254. 'chapter': '%(title)s - %(section_number)03d %(section_title)s [%(id)s].%(ext)s',
  2255. }
  2256. OUTTMPL_TYPES = {
  2257. 'chapter': None,
  2258. 'subtitle': None,
  2259. 'thumbnail': None,
  2260. 'description': 'description',
  2261. 'annotation': 'annotations.xml',
  2262. 'infojson': 'info.json',
  2263. 'link': None,
  2264. 'pl_video': None,
  2265. 'pl_thumbnail': None,
  2266. 'pl_description': 'description',
  2267. 'pl_infojson': 'info.json',
  2268. }
  2269. # As of [1] format syntax is:
  2270. # %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
  2271. # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
  2272. STR_FORMAT_RE_TMPL = r'''(?x)
  2273. (?<!%)(?P<prefix>(?:%%)*)
  2274. %
  2275. (?P<has_key>\((?P<key>{0})\))?
  2276. (?P<format>
  2277. (?P<conversion>[#0\-+ ]+)?
  2278. (?P<min_width>\d+)?
  2279. (?P<precision>\.\d+)?
  2280. (?P<len_mod>[hlL])? # unused in python
  2281. {1} # conversion type
  2282. )
  2283. '''
  2284. STR_FORMAT_TYPES = 'diouxXeEfFgGcrsa'
  2285. def limit_length(s, length):
  2286. """ Add ellipses to overly long strings """
  2287. if s is None:
  2288. return None
  2289. ELLIPSES = '...'
  2290. if len(s) > length:
  2291. return s[:length - len(ELLIPSES)] + ELLIPSES
  2292. return s
  2293. def version_tuple(v):
  2294. return tuple(int(e) for e in re.split(r'[-.]', v))
  2295. def is_outdated_version(version, limit, assume_new=True):
  2296. if not version:
  2297. return not assume_new
  2298. try:
  2299. return version_tuple(version) < version_tuple(limit)
  2300. except ValueError:
  2301. return not assume_new
  2302. def ytdl_is_updateable():
  2303. """ Returns if yt-dlp can be updated with -U """
  2304. from ..update import is_non_updateable
  2305. return not is_non_updateable()
  2306. def args_to_str(args):
  2307. # Get a short string representation for a subprocess command
  2308. return shell_quote(args)
  2309. def error_to_str(err):
  2310. return f'{type(err).__name__}: {err}'
  2311. def mimetype2ext(mt, default=NO_DEFAULT):
  2312. if not isinstance(mt, str):
  2313. if default is not NO_DEFAULT:
  2314. return default
  2315. return None
  2316. MAP = {
  2317. # video
  2318. '3gpp': '3gp',
  2319. 'mp2t': 'ts',
  2320. 'mp4': 'mp4',
  2321. 'mpeg': 'mpeg',
  2322. 'mpegurl': 'm3u8',
  2323. 'quicktime': 'mov',
  2324. 'webm': 'webm',
  2325. 'vp9': 'vp9',
  2326. 'video/ogg': 'ogv',
  2327. 'x-flv': 'flv',
  2328. 'x-m4v': 'm4v',
  2329. 'x-matroska': 'mkv',
  2330. 'x-mng': 'mng',
  2331. 'x-mp4-fragmented': 'mp4',
  2332. 'x-ms-asf': 'asf',
  2333. 'x-ms-wmv': 'wmv',
  2334. 'x-msvideo': 'avi',
  2335. # application (streaming playlists)
  2336. 'dash+xml': 'mpd',
  2337. 'f4m+xml': 'f4m',
  2338. 'hds+xml': 'f4m',
  2339. 'vnd.apple.mpegurl': 'm3u8',
  2340. 'vnd.ms-sstr+xml': 'ism',
  2341. 'x-mpegurl': 'm3u8',
  2342. # audio
  2343. 'audio/mp4': 'm4a',
  2344. # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3.
  2345. # Using .mp3 as it's the most popular one
  2346. 'audio/mpeg': 'mp3',
  2347. 'audio/webm': 'webm',
  2348. 'audio/x-matroska': 'mka',
  2349. 'audio/x-mpegurl': 'm3u',
  2350. 'aacp': 'aac',
  2351. 'midi': 'mid',
  2352. 'ogg': 'ogg',
  2353. 'wav': 'wav',
  2354. 'wave': 'wav',
  2355. 'x-aac': 'aac',
  2356. 'x-flac': 'flac',
  2357. 'x-m4a': 'm4a',
  2358. 'x-realaudio': 'ra',
  2359. 'x-wav': 'wav',
  2360. # image
  2361. 'avif': 'avif',
  2362. 'bmp': 'bmp',
  2363. 'gif': 'gif',
  2364. 'jpeg': 'jpg',
  2365. 'png': 'png',
  2366. 'svg+xml': 'svg',
  2367. 'tiff': 'tif',
  2368. 'vnd.wap.wbmp': 'wbmp',
  2369. 'webp': 'webp',
  2370. 'x-icon': 'ico',
  2371. 'x-jng': 'jng',
  2372. 'x-ms-bmp': 'bmp',
  2373. # caption
  2374. 'filmstrip+json': 'fs',
  2375. 'smptett+xml': 'tt',
  2376. 'ttaf+xml': 'dfxp',
  2377. 'ttml+xml': 'ttml',
  2378. 'x-ms-sami': 'sami',
  2379. # misc
  2380. 'gzip': 'gz',
  2381. 'json': 'json',
  2382. 'xml': 'xml',
  2383. 'zip': 'zip',
  2384. }
  2385. mimetype = mt.partition(';')[0].strip().lower()
  2386. _, _, subtype = mimetype.rpartition('/')
  2387. ext = traversal.traverse_obj(MAP, mimetype, subtype, subtype.rsplit('+')[-1])
  2388. if ext:
  2389. return ext
  2390. elif default is not NO_DEFAULT:
  2391. return default
  2392. return subtype.replace('+', '.')
  2393. def ext2mimetype(ext_or_url):
  2394. if not ext_or_url:
  2395. return None
  2396. if '.' not in ext_or_url:
  2397. ext_or_url = f'file.{ext_or_url}'
  2398. return mimetypes.guess_type(ext_or_url)[0]
  2399. def parse_codecs(codecs_str):
  2400. # http://tools.ietf.org/html/rfc6381
  2401. if not codecs_str:
  2402. return {}
  2403. split_codecs = list(filter(None, map(
  2404. str.strip, codecs_str.strip().strip(',').split(','))))
  2405. vcodec, acodec, scodec, hdr = None, None, None, None
  2406. for full_codec in split_codecs:
  2407. full_codec = re.sub(r'^([^.]+)', lambda m: m.group(1).lower(), full_codec)
  2408. parts = re.sub(r'0+(?=\d)', '', full_codec).split('.')
  2409. if parts[0] in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2',
  2410. 'h263', 'h264', 'mp4v', 'hvc1', 'av1', 'theora', 'dvh1', 'dvhe'):
  2411. if vcodec:
  2412. continue
  2413. vcodec = full_codec
  2414. if parts[0] in ('dvh1', 'dvhe'):
  2415. hdr = 'DV'
  2416. elif parts[0] == 'av1' and traversal.traverse_obj(parts, 3) == '10':
  2417. hdr = 'HDR10'
  2418. elif parts[:2] == ['vp9', '2']:
  2419. hdr = 'HDR10'
  2420. elif parts[0] in ('flac', 'mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-4',
  2421. 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
  2422. acodec = acodec or full_codec
  2423. elif parts[0] in ('stpp', 'wvtt'):
  2424. scodec = scodec or full_codec
  2425. else:
  2426. write_string(f'WARNING: Unknown codec {full_codec}\n')
  2427. if vcodec or acodec or scodec:
  2428. return {
  2429. 'vcodec': vcodec or 'none',
  2430. 'acodec': acodec or 'none',
  2431. 'dynamic_range': hdr,
  2432. **({'scodec': scodec} if scodec is not None else {}),
  2433. }
  2434. elif len(split_codecs) == 2:
  2435. return {
  2436. 'vcodec': split_codecs[0],
  2437. 'acodec': split_codecs[1],
  2438. }
  2439. return {}
  2440. def get_compatible_ext(*, vcodecs, acodecs, vexts, aexts, preferences=None):
  2441. assert len(vcodecs) == len(vexts) and len(acodecs) == len(aexts)
  2442. allow_mkv = not preferences or 'mkv' in preferences
  2443. if allow_mkv and max(len(acodecs), len(vcodecs)) > 1:
  2444. return 'mkv' # TODO: any other format allows this?
  2445. # TODO: All codecs supported by parse_codecs isn't handled here
  2446. COMPATIBLE_CODECS = {
  2447. 'mp4': {
  2448. 'av1', 'hevc', 'avc1', 'mp4a', 'ac-4', # fourcc (m3u8, mpd)
  2449. 'h264', 'aacl', 'ec-3', # Set in ISM
  2450. },
  2451. 'webm': {
  2452. 'av1', 'vp9', 'vp8', 'opus', 'vrbs',
  2453. 'vp9x', 'vp8x', # in the webm spec
  2454. },
  2455. }
  2456. sanitize_codec = functools.partial(
  2457. try_get, getter=lambda x: x[0].split('.')[0].replace('0', '').lower())
  2458. vcodec, acodec = sanitize_codec(vcodecs), sanitize_codec(acodecs)
  2459. for ext in preferences or COMPATIBLE_CODECS.keys():
  2460. codec_set = COMPATIBLE_CODECS.get(ext, set())
  2461. if ext == 'mkv' or codec_set.issuperset((vcodec, acodec)):
  2462. return ext
  2463. COMPATIBLE_EXTS = (
  2464. {'mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma', 'mov'},
  2465. {'webm', 'weba'},
  2466. )
  2467. for ext in preferences or vexts:
  2468. current_exts = {ext, *vexts, *aexts}
  2469. if ext == 'mkv' or current_exts == {ext} or any(
  2470. ext_sets.issuperset(current_exts) for ext_sets in COMPATIBLE_EXTS):
  2471. return ext
  2472. return 'mkv' if allow_mkv else preferences[-1]
  2473. def urlhandle_detect_ext(url_handle, default=NO_DEFAULT):
  2474. getheader = url_handle.headers.get
  2475. cd = getheader('Content-Disposition')
  2476. if cd:
  2477. m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
  2478. if m:
  2479. e = determine_ext(m.group('filename'), default_ext=None)
  2480. if e:
  2481. return e
  2482. meta_ext = getheader('x-amz-meta-name')
  2483. if meta_ext:
  2484. e = meta_ext.rpartition('.')[2]
  2485. if e:
  2486. return e
  2487. return mimetype2ext(getheader('Content-Type'), default=default)
  2488. def encode_data_uri(data, mime_type):
  2489. return 'data:{};base64,{}'.format(mime_type, base64.b64encode(data).decode('ascii'))
  2490. def age_restricted(content_limit, age_limit):
  2491. """ Returns True iff the content should be blocked """
  2492. if age_limit is None: # No limit set
  2493. return False
  2494. if content_limit is None:
  2495. return False # Content available for everyone
  2496. return age_limit < content_limit
  2497. # List of known byte-order-marks (BOM)
  2498. BOMS = [
  2499. (b'\xef\xbb\xbf', 'utf-8'),
  2500. (b'\x00\x00\xfe\xff', 'utf-32-be'),
  2501. (b'\xff\xfe\x00\x00', 'utf-32-le'),
  2502. (b'\xff\xfe', 'utf-16-le'),
  2503. (b'\xfe\xff', 'utf-16-be'),
  2504. ]
  2505. def is_html(first_bytes):
  2506. """ Detect whether a file contains HTML by examining its first bytes. """
  2507. encoding = 'utf-8'
  2508. for bom, enc in BOMS:
  2509. while first_bytes.startswith(bom):
  2510. encoding, first_bytes = enc, first_bytes[len(bom):]
  2511. return re.match(r'\s*<', first_bytes.decode(encoding, 'replace'))
  2512. def determine_protocol(info_dict):
  2513. protocol = info_dict.get('protocol')
  2514. if protocol is not None:
  2515. return protocol
  2516. url = sanitize_url(info_dict['url'])
  2517. if url.startswith('rtmp'):
  2518. return 'rtmp'
  2519. elif url.startswith('mms'):
  2520. return 'mms'
  2521. elif url.startswith('rtsp'):
  2522. return 'rtsp'
  2523. ext = determine_ext(url)
  2524. if ext == 'm3u8':
  2525. return 'm3u8' if info_dict.get('is_live') else 'm3u8_native'
  2526. elif ext == 'f4m':
  2527. return 'f4m'
  2528. return urllib.parse.urlparse(url).scheme
  2529. def render_table(header_row, data, delim=False, extra_gap=0, hide_empty=False):
  2530. """ Render a list of rows, each as a list of values.
  2531. Text after a \t will be right aligned """
  2532. def width(string):
  2533. return len(remove_terminal_sequences(string).replace('\t', ''))
  2534. def get_max_lens(table):
  2535. return [max(width(str(v)) for v in col) for col in zip(*table)]
  2536. def filter_using_list(row, filter_array):
  2537. return [col for take, col in itertools.zip_longest(filter_array, row, fillvalue=True) if take]
  2538. max_lens = get_max_lens(data) if hide_empty else []
  2539. header_row = filter_using_list(header_row, max_lens)
  2540. data = [filter_using_list(row, max_lens) for row in data]
  2541. table = [header_row, *data]
  2542. max_lens = get_max_lens(table)
  2543. extra_gap += 1
  2544. if delim:
  2545. table = [header_row, [delim * (ml + extra_gap) for ml in max_lens], *data]
  2546. table[1][-1] = table[1][-1][:-extra_gap * len(delim)] # Remove extra_gap from end of delimiter
  2547. for row in table:
  2548. for pos, text in enumerate(map(str, row)):
  2549. if '\t' in text:
  2550. row[pos] = text.replace('\t', ' ' * (max_lens[pos] - width(text))) + ' ' * extra_gap
  2551. else:
  2552. row[pos] = text + ' ' * (max_lens[pos] - width(text) + extra_gap)
  2553. return '\n'.join(''.join(row).rstrip() for row in table)
  2554. def _match_one(filter_part, dct, incomplete):
  2555. # TODO: Generalize code with YoutubeDL._build_format_filter
  2556. STRING_OPERATORS = {
  2557. '*=': operator.contains,
  2558. '^=': lambda attr, value: attr.startswith(value),
  2559. '$=': lambda attr, value: attr.endswith(value),
  2560. '~=': lambda attr, value: re.search(value, attr),
  2561. }
  2562. COMPARISON_OPERATORS = {
  2563. **STRING_OPERATORS,
  2564. '<=': operator.le, # "<=" must be defined above "<"
  2565. '<': operator.lt,
  2566. '>=': operator.ge,
  2567. '>': operator.gt,
  2568. '=': operator.eq,
  2569. }
  2570. if isinstance(incomplete, bool):
  2571. is_incomplete = lambda _: incomplete
  2572. else:
  2573. is_incomplete = lambda k: k in incomplete
  2574. operator_rex = re.compile(r'''(?x)
  2575. (?P<key>[a-z_]+)
  2576. \s*(?P<negation>!\s*)?(?P<op>{})(?P<none_inclusive>\s*\?)?\s*
  2577. (?:
  2578. (?P<quote>["\'])(?P<quotedstrval>.+?)(?P=quote)|
  2579. (?P<strval>.+?)
  2580. )
  2581. '''.format('|'.join(map(re.escape, COMPARISON_OPERATORS.keys()))))
  2582. m = operator_rex.fullmatch(filter_part.strip())
  2583. if m:
  2584. m = m.groupdict()
  2585. unnegated_op = COMPARISON_OPERATORS[m['op']]
  2586. if m['negation']:
  2587. op = lambda attr, value: not unnegated_op(attr, value)
  2588. else:
  2589. op = unnegated_op
  2590. comparison_value = m['quotedstrval'] or m['strval'] or m['intval']
  2591. if m['quote']:
  2592. comparison_value = comparison_value.replace(r'\{}'.format(m['quote']), m['quote'])
  2593. actual_value = dct.get(m['key'])
  2594. numeric_comparison = None
  2595. if isinstance(actual_value, (int, float)):
  2596. # If the original field is a string and matching comparisonvalue is
  2597. # a number we should respect the origin of the original field
  2598. # and process comparison value as a string (see
  2599. # https://github.com/ytdl-org/youtube-dl/issues/11082)
  2600. try:
  2601. numeric_comparison = int(comparison_value)
  2602. except ValueError:
  2603. numeric_comparison = parse_filesize(comparison_value)
  2604. if numeric_comparison is None:
  2605. numeric_comparison = parse_filesize(f'{comparison_value}B')
  2606. if numeric_comparison is None:
  2607. numeric_comparison = parse_duration(comparison_value)
  2608. if numeric_comparison is not None and m['op'] in STRING_OPERATORS:
  2609. raise ValueError('Operator {} only supports string values!'.format(m['op']))
  2610. if actual_value is None:
  2611. return is_incomplete(m['key']) or m['none_inclusive']
  2612. return op(actual_value, comparison_value if numeric_comparison is None else numeric_comparison)
  2613. UNARY_OPERATORS = {
  2614. '': lambda v: (v is True) if isinstance(v, bool) else (v is not None),
  2615. '!': lambda v: (v is False) if isinstance(v, bool) else (v is None),
  2616. }
  2617. operator_rex = re.compile(r'''(?x)
  2618. (?P<op>{})\s*(?P<key>[a-z_]+)
  2619. '''.format('|'.join(map(re.escape, UNARY_OPERATORS.keys()))))
  2620. m = operator_rex.fullmatch(filter_part.strip())
  2621. if m:
  2622. op = UNARY_OPERATORS[m.group('op')]
  2623. actual_value = dct.get(m.group('key'))
  2624. if is_incomplete(m.group('key')) and actual_value is None:
  2625. return True
  2626. return op(actual_value)
  2627. raise ValueError(f'Invalid filter part {filter_part!r}')
  2628. def match_str(filter_str, dct, incomplete=False):
  2629. """ Filter a dictionary with a simple string syntax.
  2630. @returns Whether the filter passes
  2631. @param incomplete Set of keys that is expected to be missing from dct.
  2632. Can be True/False to indicate all/none of the keys may be missing.
  2633. All conditions on incomplete keys pass if the key is missing
  2634. """
  2635. return all(
  2636. _match_one(filter_part.replace(r'\&', '&'), dct, incomplete)
  2637. for filter_part in re.split(r'(?<!\\)&', filter_str))
  2638. def match_filter_func(filters, breaking_filters=None):
  2639. if not filters and not breaking_filters:
  2640. return None
  2641. repr_ = f'{match_filter_func.__module__}.{match_filter_func.__qualname__}({filters}, {breaking_filters})'
  2642. breaking_filters = match_filter_func(breaking_filters) or (lambda _, __: None)
  2643. filters = set(variadic(filters or []))
  2644. interactive = '-' in filters
  2645. if interactive:
  2646. filters.remove('-')
  2647. @function_with_repr.set_repr(repr_)
  2648. def _match_func(info_dict, incomplete=False):
  2649. ret = breaking_filters(info_dict, incomplete)
  2650. if ret is not None:
  2651. raise RejectedVideoReached(ret)
  2652. if not filters or any(match_str(f, info_dict, incomplete) for f in filters):
  2653. return NO_DEFAULT if interactive and not incomplete else None
  2654. else:
  2655. video_title = info_dict.get('title') or info_dict.get('id') or 'entry'
  2656. filter_str = ') | ('.join(map(str.strip, filters))
  2657. return f'{video_title} does not pass filter ({filter_str}), skipping ..'
  2658. return _match_func
  2659. class download_range_func:
  2660. def __init__(self, chapters, ranges, from_info=False):
  2661. self.chapters, self.ranges, self.from_info = chapters, ranges, from_info
  2662. def __call__(self, info_dict, ydl):
  2663. warning = ('There are no chapters matching the regex' if info_dict.get('chapters')
  2664. else 'Cannot match chapters since chapter information is unavailable')
  2665. for regex in self.chapters or []:
  2666. for i, chapter in enumerate(info_dict.get('chapters') or []):
  2667. if re.search(regex, chapter['title']):
  2668. warning = None
  2669. yield {**chapter, 'index': i}
  2670. if self.chapters and warning:
  2671. ydl.to_screen(f'[info] {info_dict["id"]}: {warning}')
  2672. for start, end in self.ranges or []:
  2673. yield {
  2674. 'start_time': self._handle_negative_timestamp(start, info_dict),
  2675. 'end_time': self._handle_negative_timestamp(end, info_dict),
  2676. }
  2677. if self.from_info and (info_dict.get('start_time') or info_dict.get('end_time')):
  2678. yield {
  2679. 'start_time': info_dict.get('start_time') or 0,
  2680. 'end_time': info_dict.get('end_time') or float('inf'),
  2681. }
  2682. elif not self.ranges and not self.chapters:
  2683. yield {}
  2684. @staticmethod
  2685. def _handle_negative_timestamp(time, info):
  2686. return max(info['duration'] + time, 0) if info.get('duration') and time < 0 else time
  2687. def __eq__(self, other):
  2688. return (isinstance(other, download_range_func)
  2689. and self.chapters == other.chapters and self.ranges == other.ranges)
  2690. def __repr__(self):
  2691. return f'{__name__}.{type(self).__name__}({self.chapters}, {self.ranges})'
  2692. def parse_dfxp_time_expr(time_expr):
  2693. if not time_expr:
  2694. return
  2695. mobj = re.match(rf'^(?P<time_offset>{NUMBER_RE})s?$', time_expr)
  2696. if mobj:
  2697. return float(mobj.group('time_offset'))
  2698. mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
  2699. if mobj:
  2700. return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
  2701. def srt_subtitles_timecode(seconds):
  2702. return '%02d:%02d:%02d,%03d' % timetuple_from_msec(seconds * 1000)
  2703. def ass_subtitles_timecode(seconds):
  2704. time = timetuple_from_msec(seconds * 1000)
  2705. return '%01d:%02d:%02d.%02d' % (*time[:-1], time.milliseconds / 10)
  2706. def dfxp2srt(dfxp_data):
  2707. """
  2708. @param dfxp_data A bytes-like object containing DFXP data
  2709. @returns A unicode object containing converted SRT data
  2710. """
  2711. LEGACY_NAMESPACES = (
  2712. (b'http://www.w3.org/ns/ttml', [
  2713. b'http://www.w3.org/2004/11/ttaf1',
  2714. b'http://www.w3.org/2006/04/ttaf1',
  2715. b'http://www.w3.org/2006/10/ttaf1',
  2716. ]),
  2717. (b'http://www.w3.org/ns/ttml#styling', [
  2718. b'http://www.w3.org/ns/ttml#style',
  2719. ]),
  2720. )
  2721. SUPPORTED_STYLING = [
  2722. 'color',
  2723. 'fontFamily',
  2724. 'fontSize',
  2725. 'fontStyle',
  2726. 'fontWeight',
  2727. 'textDecoration',
  2728. ]
  2729. _x = functools.partial(xpath_with_ns, ns_map={
  2730. 'xml': 'http://www.w3.org/XML/1998/namespace',
  2731. 'ttml': 'http://www.w3.org/ns/ttml',
  2732. 'tts': 'http://www.w3.org/ns/ttml#styling',
  2733. })
  2734. styles = {}
  2735. default_style = {}
  2736. class TTMLPElementParser:
  2737. _out = ''
  2738. _unclosed_elements = []
  2739. _applied_styles = []
  2740. def start(self, tag, attrib):
  2741. if tag in (_x('ttml:br'), 'br'):
  2742. self._out += '\n'
  2743. else:
  2744. unclosed_elements = []
  2745. style = {}
  2746. element_style_id = attrib.get('style')
  2747. if default_style:
  2748. style.update(default_style)
  2749. if element_style_id:
  2750. style.update(styles.get(element_style_id, {}))
  2751. for prop in SUPPORTED_STYLING:
  2752. prop_val = attrib.get(_x('tts:' + prop))
  2753. if prop_val:
  2754. style[prop] = prop_val
  2755. if style:
  2756. font = ''
  2757. for k, v in sorted(style.items()):
  2758. if self._applied_styles and self._applied_styles[-1].get(k) == v:
  2759. continue
  2760. if k == 'color':
  2761. font += f' color="{v}"'
  2762. elif k == 'fontSize':
  2763. font += f' size="{v}"'
  2764. elif k == 'fontFamily':
  2765. font += f' face="{v}"'
  2766. elif k == 'fontWeight' and v == 'bold':
  2767. self._out += '<b>'
  2768. unclosed_elements.append('b')
  2769. elif k == 'fontStyle' and v == 'italic':
  2770. self._out += '<i>'
  2771. unclosed_elements.append('i')
  2772. elif k == 'textDecoration' and v == 'underline':
  2773. self._out += '<u>'
  2774. unclosed_elements.append('u')
  2775. if font:
  2776. self._out += '<font' + font + '>'
  2777. unclosed_elements.append('font')
  2778. applied_style = {}
  2779. if self._applied_styles:
  2780. applied_style.update(self._applied_styles[-1])
  2781. applied_style.update(style)
  2782. self._applied_styles.append(applied_style)
  2783. self._unclosed_elements.append(unclosed_elements)
  2784. def end(self, tag):
  2785. if tag not in (_x('ttml:br'), 'br'):
  2786. unclosed_elements = self._unclosed_elements.pop()
  2787. for element in reversed(unclosed_elements):
  2788. self._out += f'</{element}>'
  2789. if unclosed_elements and self._applied_styles:
  2790. self._applied_styles.pop()
  2791. def data(self, data):
  2792. self._out += data
  2793. def close(self):
  2794. return self._out.strip()
  2795. # Fix UTF-8 encoded file wrongly marked as UTF-16. See https://github.com/yt-dlp/yt-dlp/issues/6543#issuecomment-1477169870
  2796. # This will not trigger false positives since only UTF-8 text is being replaced
  2797. dfxp_data = dfxp_data.replace(b'encoding=\'UTF-16\'', b'encoding=\'UTF-8\'')
  2798. def parse_node(node):
  2799. target = TTMLPElementParser()
  2800. parser = xml.etree.ElementTree.XMLParser(target=target)
  2801. parser.feed(xml.etree.ElementTree.tostring(node))
  2802. return parser.close()
  2803. for k, v in LEGACY_NAMESPACES:
  2804. for ns in v:
  2805. dfxp_data = dfxp_data.replace(ns, k)
  2806. dfxp = compat_etree_fromstring(dfxp_data)
  2807. out = []
  2808. paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
  2809. if not paras:
  2810. raise ValueError('Invalid dfxp/TTML subtitle')
  2811. repeat = False
  2812. while True:
  2813. for style in dfxp.findall(_x('.//ttml:style')):
  2814. style_id = style.get('id') or style.get(_x('xml:id'))
  2815. if not style_id:
  2816. continue
  2817. parent_style_id = style.get('style')
  2818. if parent_style_id:
  2819. if parent_style_id not in styles:
  2820. repeat = True
  2821. continue
  2822. styles[style_id] = styles[parent_style_id].copy()
  2823. for prop in SUPPORTED_STYLING:
  2824. prop_val = style.get(_x('tts:' + prop))
  2825. if prop_val:
  2826. styles.setdefault(style_id, {})[prop] = prop_val
  2827. if repeat:
  2828. repeat = False
  2829. else:
  2830. break
  2831. for p in ('body', 'div'):
  2832. ele = xpath_element(dfxp, [_x('.//ttml:' + p), './/' + p])
  2833. if ele is None:
  2834. continue
  2835. style = styles.get(ele.get('style'))
  2836. if not style:
  2837. continue
  2838. default_style.update(style)
  2839. for para, index in zip(paras, itertools.count(1)):
  2840. begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
  2841. end_time = parse_dfxp_time_expr(para.attrib.get('end'))
  2842. dur = parse_dfxp_time_expr(para.attrib.get('dur'))
  2843. if begin_time is None:
  2844. continue
  2845. if not end_time:
  2846. if not dur:
  2847. continue
  2848. end_time = begin_time + dur
  2849. out.append('%d\n%s --> %s\n%s\n\n' % (
  2850. index,
  2851. srt_subtitles_timecode(begin_time),
  2852. srt_subtitles_timecode(end_time),
  2853. parse_node(para)))
  2854. return ''.join(out)
  2855. def cli_option(params, command_option, param, separator=None):
  2856. param = params.get(param)
  2857. return ([] if param is None
  2858. else [command_option, str(param)] if separator is None
  2859. else [f'{command_option}{separator}{param}'])
  2860. def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
  2861. param = params.get(param)
  2862. assert param in (True, False, None)
  2863. return cli_option({True: true_value, False: false_value}, command_option, param, separator)
  2864. def cli_valueless_option(params, command_option, param, expected_value=True):
  2865. return [command_option] if params.get(param) == expected_value else []
  2866. def cli_configuration_args(argdict, keys, default=[], use_compat=True):
  2867. if isinstance(argdict, (list, tuple)): # for backward compatibility
  2868. if use_compat:
  2869. return argdict
  2870. else:
  2871. argdict = None
  2872. if argdict is None:
  2873. return default
  2874. assert isinstance(argdict, dict)
  2875. assert isinstance(keys, (list, tuple))
  2876. for key_list in keys:
  2877. arg_list = list(filter(
  2878. lambda x: x is not None,
  2879. [argdict.get(key.lower()) for key in variadic(key_list)]))
  2880. if arg_list:
  2881. return [arg for args in arg_list for arg in args]
  2882. return default
  2883. def _configuration_args(main_key, argdict, exe, keys=None, default=[], use_compat=True):
  2884. main_key, exe = main_key.lower(), exe.lower()
  2885. root_key = exe if main_key == exe else f'{main_key}+{exe}'
  2886. keys = [f'{root_key}{k}' for k in (keys or [''])]
  2887. if root_key in keys:
  2888. if main_key != exe:
  2889. keys.append((main_key, exe))
  2890. keys.append('default')
  2891. else:
  2892. use_compat = False
  2893. return cli_configuration_args(argdict, keys, default, use_compat)
  2894. class ISO639Utils:
  2895. # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
  2896. _lang_map = {
  2897. 'aa': 'aar',
  2898. 'ab': 'abk',
  2899. 'ae': 'ave',
  2900. 'af': 'afr',
  2901. 'ak': 'aka',
  2902. 'am': 'amh',
  2903. 'an': 'arg',
  2904. 'ar': 'ara',
  2905. 'as': 'asm',
  2906. 'av': 'ava',
  2907. 'ay': 'aym',
  2908. 'az': 'aze',
  2909. 'ba': 'bak',
  2910. 'be': 'bel',
  2911. 'bg': 'bul',
  2912. 'bh': 'bih',
  2913. 'bi': 'bis',
  2914. 'bm': 'bam',
  2915. 'bn': 'ben',
  2916. 'bo': 'bod',
  2917. 'br': 'bre',
  2918. 'bs': 'bos',
  2919. 'ca': 'cat',
  2920. 'ce': 'che',
  2921. 'ch': 'cha',
  2922. 'co': 'cos',
  2923. 'cr': 'cre',
  2924. 'cs': 'ces',
  2925. 'cu': 'chu',
  2926. 'cv': 'chv',
  2927. 'cy': 'cym',
  2928. 'da': 'dan',
  2929. 'de': 'deu',
  2930. 'dv': 'div',
  2931. 'dz': 'dzo',
  2932. 'ee': 'ewe',
  2933. 'el': 'ell',
  2934. 'en': 'eng',
  2935. 'eo': 'epo',
  2936. 'es': 'spa',
  2937. 'et': 'est',
  2938. 'eu': 'eus',
  2939. 'fa': 'fas',
  2940. 'ff': 'ful',
  2941. 'fi': 'fin',
  2942. 'fj': 'fij',
  2943. 'fo': 'fao',
  2944. 'fr': 'fra',
  2945. 'fy': 'fry',
  2946. 'ga': 'gle',
  2947. 'gd': 'gla',
  2948. 'gl': 'glg',
  2949. 'gn': 'grn',
  2950. 'gu': 'guj',
  2951. 'gv': 'glv',
  2952. 'ha': 'hau',
  2953. 'he': 'heb',
  2954. 'iw': 'heb', # Replaced by he in 1989 revision
  2955. 'hi': 'hin',
  2956. 'ho': 'hmo',
  2957. 'hr': 'hrv',
  2958. 'ht': 'hat',
  2959. 'hu': 'hun',
  2960. 'hy': 'hye',
  2961. 'hz': 'her',
  2962. 'ia': 'ina',
  2963. 'id': 'ind',
  2964. 'in': 'ind', # Replaced by id in 1989 revision
  2965. 'ie': 'ile',
  2966. 'ig': 'ibo',
  2967. 'ii': 'iii',
  2968. 'ik': 'ipk',
  2969. 'io': 'ido',
  2970. 'is': 'isl',
  2971. 'it': 'ita',
  2972. 'iu': 'iku',
  2973. 'ja': 'jpn',
  2974. 'jv': 'jav',
  2975. 'ka': 'kat',
  2976. 'kg': 'kon',
  2977. 'ki': 'kik',
  2978. 'kj': 'kua',
  2979. 'kk': 'kaz',
  2980. 'kl': 'kal',
  2981. 'km': 'khm',
  2982. 'kn': 'kan',
  2983. 'ko': 'kor',
  2984. 'kr': 'kau',
  2985. 'ks': 'kas',
  2986. 'ku': 'kur',
  2987. 'kv': 'kom',
  2988. 'kw': 'cor',
  2989. 'ky': 'kir',
  2990. 'la': 'lat',
  2991. 'lb': 'ltz',
  2992. 'lg': 'lug',
  2993. 'li': 'lim',
  2994. 'ln': 'lin',
  2995. 'lo': 'lao',
  2996. 'lt': 'lit',
  2997. 'lu': 'lub',
  2998. 'lv': 'lav',
  2999. 'mg': 'mlg',
  3000. 'mh': 'mah',
  3001. 'mi': 'mri',
  3002. 'mk': 'mkd',
  3003. 'ml': 'mal',
  3004. 'mn': 'mon',
  3005. 'mr': 'mar',
  3006. 'ms': 'msa',
  3007. 'mt': 'mlt',
  3008. 'my': 'mya',
  3009. 'na': 'nau',
  3010. 'nb': 'nob',
  3011. 'nd': 'nde',
  3012. 'ne': 'nep',
  3013. 'ng': 'ndo',
  3014. 'nl': 'nld',
  3015. 'nn': 'nno',
  3016. 'no': 'nor',
  3017. 'nr': 'nbl',
  3018. 'nv': 'nav',
  3019. 'ny': 'nya',
  3020. 'oc': 'oci',
  3021. 'oj': 'oji',
  3022. 'om': 'orm',
  3023. 'or': 'ori',
  3024. 'os': 'oss',
  3025. 'pa': 'pan',
  3026. 'pe': 'per',
  3027. 'pi': 'pli',
  3028. 'pl': 'pol',
  3029. 'ps': 'pus',
  3030. 'pt': 'por',
  3031. 'qu': 'que',
  3032. 'rm': 'roh',
  3033. 'rn': 'run',
  3034. 'ro': 'ron',
  3035. 'ru': 'rus',
  3036. 'rw': 'kin',
  3037. 'sa': 'san',
  3038. 'sc': 'srd',
  3039. 'sd': 'snd',
  3040. 'se': 'sme',
  3041. 'sg': 'sag',
  3042. 'si': 'sin',
  3043. 'sk': 'slk',
  3044. 'sl': 'slv',
  3045. 'sm': 'smo',
  3046. 'sn': 'sna',
  3047. 'so': 'som',
  3048. 'sq': 'sqi',
  3049. 'sr': 'srp',
  3050. 'ss': 'ssw',
  3051. 'st': 'sot',
  3052. 'su': 'sun',
  3053. 'sv': 'swe',
  3054. 'sw': 'swa',
  3055. 'ta': 'tam',
  3056. 'te': 'tel',
  3057. 'tg': 'tgk',
  3058. 'th': 'tha',
  3059. 'ti': 'tir',
  3060. 'tk': 'tuk',
  3061. 'tl': 'tgl',
  3062. 'tn': 'tsn',
  3063. 'to': 'ton',
  3064. 'tr': 'tur',
  3065. 'ts': 'tso',
  3066. 'tt': 'tat',
  3067. 'tw': 'twi',
  3068. 'ty': 'tah',
  3069. 'ug': 'uig',
  3070. 'uk': 'ukr',
  3071. 'ur': 'urd',
  3072. 'uz': 'uzb',
  3073. 've': 'ven',
  3074. 'vi': 'vie',
  3075. 'vo': 'vol',
  3076. 'wa': 'wln',
  3077. 'wo': 'wol',
  3078. 'xh': 'xho',
  3079. 'yi': 'yid',
  3080. 'ji': 'yid', # Replaced by yi in 1989 revision
  3081. 'yo': 'yor',
  3082. 'za': 'zha',
  3083. 'zh': 'zho',
  3084. 'zu': 'zul',
  3085. }
  3086. @classmethod
  3087. def short2long(cls, code):
  3088. """Convert language code from ISO 639-1 to ISO 639-2/T"""
  3089. return cls._lang_map.get(code[:2])
  3090. @classmethod
  3091. def long2short(cls, code):
  3092. """Convert language code from ISO 639-2/T to ISO 639-1"""
  3093. for short_name, long_name in cls._lang_map.items():
  3094. if long_name == code:
  3095. return short_name
  3096. class ISO3166Utils:
  3097. # From http://data.okfn.org/data/core/country-list
  3098. _country_map = {
  3099. 'AF': 'Afghanistan',
  3100. 'AX': 'Åland Islands',
  3101. 'AL': 'Albania',
  3102. 'DZ': 'Algeria',
  3103. 'AS': 'American Samoa',
  3104. 'AD': 'Andorra',
  3105. 'AO': 'Angola',
  3106. 'AI': 'Anguilla',
  3107. 'AQ': 'Antarctica',
  3108. 'AG': 'Antigua and Barbuda',
  3109. 'AR': 'Argentina',
  3110. 'AM': 'Armenia',
  3111. 'AW': 'Aruba',
  3112. 'AU': 'Australia',
  3113. 'AT': 'Austria',
  3114. 'AZ': 'Azerbaijan',
  3115. 'BS': 'Bahamas',
  3116. 'BH': 'Bahrain',
  3117. 'BD': 'Bangladesh',
  3118. 'BB': 'Barbados',
  3119. 'BY': 'Belarus',
  3120. 'BE': 'Belgium',
  3121. 'BZ': 'Belize',
  3122. 'BJ': 'Benin',
  3123. 'BM': 'Bermuda',
  3124. 'BT': 'Bhutan',
  3125. 'BO': 'Bolivia, Plurinational State of',
  3126. 'BQ': 'Bonaire, Sint Eustatius and Saba',
  3127. 'BA': 'Bosnia and Herzegovina',
  3128. 'BW': 'Botswana',
  3129. 'BV': 'Bouvet Island',
  3130. 'BR': 'Brazil',
  3131. 'IO': 'British Indian Ocean Territory',
  3132. 'BN': 'Brunei Darussalam',
  3133. 'BG': 'Bulgaria',
  3134. 'BF': 'Burkina Faso',
  3135. 'BI': 'Burundi',
  3136. 'KH': 'Cambodia',
  3137. 'CM': 'Cameroon',
  3138. 'CA': 'Canada',
  3139. 'CV': 'Cape Verde',
  3140. 'KY': 'Cayman Islands',
  3141. 'CF': 'Central African Republic',
  3142. 'TD': 'Chad',
  3143. 'CL': 'Chile',
  3144. 'CN': 'China',
  3145. 'CX': 'Christmas Island',
  3146. 'CC': 'Cocos (Keeling) Islands',
  3147. 'CO': 'Colombia',
  3148. 'KM': 'Comoros',
  3149. 'CG': 'Congo',
  3150. 'CD': 'Congo, the Democratic Republic of the',
  3151. 'CK': 'Cook Islands',
  3152. 'CR': 'Costa Rica',
  3153. 'CI': 'Côte d\'Ivoire',
  3154. 'HR': 'Croatia',
  3155. 'CU': 'Cuba',
  3156. 'CW': 'Curaçao',
  3157. 'CY': 'Cyprus',
  3158. 'CZ': 'Czech Republic',
  3159. 'DK': 'Denmark',
  3160. 'DJ': 'Djibouti',
  3161. 'DM': 'Dominica',
  3162. 'DO': 'Dominican Republic',
  3163. 'EC': 'Ecuador',
  3164. 'EG': 'Egypt',
  3165. 'SV': 'El Salvador',
  3166. 'GQ': 'Equatorial Guinea',
  3167. 'ER': 'Eritrea',
  3168. 'EE': 'Estonia',
  3169. 'ET': 'Ethiopia',
  3170. 'FK': 'Falkland Islands (Malvinas)',
  3171. 'FO': 'Faroe Islands',
  3172. 'FJ': 'Fiji',
  3173. 'FI': 'Finland',
  3174. 'FR': 'France',
  3175. 'GF': 'French Guiana',
  3176. 'PF': 'French Polynesia',
  3177. 'TF': 'French Southern Territories',
  3178. 'GA': 'Gabon',
  3179. 'GM': 'Gambia',
  3180. 'GE': 'Georgia',
  3181. 'DE': 'Germany',
  3182. 'GH': 'Ghana',
  3183. 'GI': 'Gibraltar',
  3184. 'GR': 'Greece',
  3185. 'GL': 'Greenland',
  3186. 'GD': 'Grenada',
  3187. 'GP': 'Guadeloupe',
  3188. 'GU': 'Guam',
  3189. 'GT': 'Guatemala',
  3190. 'GG': 'Guernsey',
  3191. 'GN': 'Guinea',
  3192. 'GW': 'Guinea-Bissau',
  3193. 'GY': 'Guyana',
  3194. 'HT': 'Haiti',
  3195. 'HM': 'Heard Island and McDonald Islands',
  3196. 'VA': 'Holy See (Vatican City State)',
  3197. 'HN': 'Honduras',
  3198. 'HK': 'Hong Kong',
  3199. 'HU': 'Hungary',
  3200. 'IS': 'Iceland',
  3201. 'IN': 'India',
  3202. 'ID': 'Indonesia',
  3203. 'IR': 'Iran, Islamic Republic of',
  3204. 'IQ': 'Iraq',
  3205. 'IE': 'Ireland',
  3206. 'IM': 'Isle of Man',
  3207. 'IL': 'Israel',
  3208. 'IT': 'Italy',
  3209. 'JM': 'Jamaica',
  3210. 'JP': 'Japan',
  3211. 'JE': 'Jersey',
  3212. 'JO': 'Jordan',
  3213. 'KZ': 'Kazakhstan',
  3214. 'KE': 'Kenya',
  3215. 'KI': 'Kiribati',
  3216. 'KP': 'Korea, Democratic People\'s Republic of',
  3217. 'KR': 'Korea, Republic of',
  3218. 'KW': 'Kuwait',
  3219. 'KG': 'Kyrgyzstan',
  3220. 'LA': 'Lao People\'s Democratic Republic',
  3221. 'LV': 'Latvia',
  3222. 'LB': 'Lebanon',
  3223. 'LS': 'Lesotho',
  3224. 'LR': 'Liberia',
  3225. 'LY': 'Libya',
  3226. 'LI': 'Liechtenstein',
  3227. 'LT': 'Lithuania',
  3228. 'LU': 'Luxembourg',
  3229. 'MO': 'Macao',
  3230. 'MK': 'Macedonia, the Former Yugoslav Republic of',
  3231. 'MG': 'Madagascar',
  3232. 'MW': 'Malawi',
  3233. 'MY': 'Malaysia',
  3234. 'MV': 'Maldives',
  3235. 'ML': 'Mali',
  3236. 'MT': 'Malta',
  3237. 'MH': 'Marshall Islands',
  3238. 'MQ': 'Martinique',
  3239. 'MR': 'Mauritania',
  3240. 'MU': 'Mauritius',
  3241. 'YT': 'Mayotte',
  3242. 'MX': 'Mexico',
  3243. 'FM': 'Micronesia, Federated States of',
  3244. 'MD': 'Moldova, Republic of',
  3245. 'MC': 'Monaco',
  3246. 'MN': 'Mongolia',
  3247. 'ME': 'Montenegro',
  3248. 'MS': 'Montserrat',
  3249. 'MA': 'Morocco',
  3250. 'MZ': 'Mozambique',
  3251. 'MM': 'Myanmar',
  3252. 'NA': 'Namibia',
  3253. 'NR': 'Nauru',
  3254. 'NP': 'Nepal',
  3255. 'NL': 'Netherlands',
  3256. 'NC': 'New Caledonia',
  3257. 'NZ': 'New Zealand',
  3258. 'NI': 'Nicaragua',
  3259. 'NE': 'Niger',
  3260. 'NG': 'Nigeria',
  3261. 'NU': 'Niue',
  3262. 'NF': 'Norfolk Island',
  3263. 'MP': 'Northern Mariana Islands',
  3264. 'NO': 'Norway',
  3265. 'OM': 'Oman',
  3266. 'PK': 'Pakistan',
  3267. 'PW': 'Palau',
  3268. 'PS': 'Palestine, State of',
  3269. 'PA': 'Panama',
  3270. 'PG': 'Papua New Guinea',
  3271. 'PY': 'Paraguay',
  3272. 'PE': 'Peru',
  3273. 'PH': 'Philippines',
  3274. 'PN': 'Pitcairn',
  3275. 'PL': 'Poland',
  3276. 'PT': 'Portugal',
  3277. 'PR': 'Puerto Rico',
  3278. 'QA': 'Qatar',
  3279. 'RE': 'Réunion',
  3280. 'RO': 'Romania',
  3281. 'RU': 'Russian Federation',
  3282. 'RW': 'Rwanda',
  3283. 'BL': 'Saint Barthélemy',
  3284. 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
  3285. 'KN': 'Saint Kitts and Nevis',
  3286. 'LC': 'Saint Lucia',
  3287. 'MF': 'Saint Martin (French part)',
  3288. 'PM': 'Saint Pierre and Miquelon',
  3289. 'VC': 'Saint Vincent and the Grenadines',
  3290. 'WS': 'Samoa',
  3291. 'SM': 'San Marino',
  3292. 'ST': 'Sao Tome and Principe',
  3293. 'SA': 'Saudi Arabia',
  3294. 'SN': 'Senegal',
  3295. 'RS': 'Serbia',
  3296. 'SC': 'Seychelles',
  3297. 'SL': 'Sierra Leone',
  3298. 'SG': 'Singapore',
  3299. 'SX': 'Sint Maarten (Dutch part)',
  3300. 'SK': 'Slovakia',
  3301. 'SI': 'Slovenia',
  3302. 'SB': 'Solomon Islands',
  3303. 'SO': 'Somalia',
  3304. 'ZA': 'South Africa',
  3305. 'GS': 'South Georgia and the South Sandwich Islands',
  3306. 'SS': 'South Sudan',
  3307. 'ES': 'Spain',
  3308. 'LK': 'Sri Lanka',
  3309. 'SD': 'Sudan',
  3310. 'SR': 'Suriname',
  3311. 'SJ': 'Svalbard and Jan Mayen',
  3312. 'SZ': 'Swaziland',
  3313. 'SE': 'Sweden',
  3314. 'CH': 'Switzerland',
  3315. 'SY': 'Syrian Arab Republic',
  3316. 'TW': 'Taiwan, Province of China',
  3317. 'TJ': 'Tajikistan',
  3318. 'TZ': 'Tanzania, United Republic of',
  3319. 'TH': 'Thailand',
  3320. 'TL': 'Timor-Leste',
  3321. 'TG': 'Togo',
  3322. 'TK': 'Tokelau',
  3323. 'TO': 'Tonga',
  3324. 'TT': 'Trinidad and Tobago',
  3325. 'TN': 'Tunisia',
  3326. 'TR': 'Turkey',
  3327. 'TM': 'Turkmenistan',
  3328. 'TC': 'Turks and Caicos Islands',
  3329. 'TV': 'Tuvalu',
  3330. 'UG': 'Uganda',
  3331. 'UA': 'Ukraine',
  3332. 'AE': 'United Arab Emirates',
  3333. 'GB': 'United Kingdom',
  3334. 'US': 'United States',
  3335. 'UM': 'United States Minor Outlying Islands',
  3336. 'UY': 'Uruguay',
  3337. 'UZ': 'Uzbekistan',
  3338. 'VU': 'Vanuatu',
  3339. 'VE': 'Venezuela, Bolivarian Republic of',
  3340. 'VN': 'Viet Nam',
  3341. 'VG': 'Virgin Islands, British',
  3342. 'VI': 'Virgin Islands, U.S.',
  3343. 'WF': 'Wallis and Futuna',
  3344. 'EH': 'Western Sahara',
  3345. 'YE': 'Yemen',
  3346. 'ZM': 'Zambia',
  3347. 'ZW': 'Zimbabwe',
  3348. # Not ISO 3166 codes, but used for IP blocks
  3349. 'AP': 'Asia/Pacific Region',
  3350. 'EU': 'Europe',
  3351. }
  3352. @classmethod
  3353. def short2full(cls, code):
  3354. """Convert an ISO 3166-2 country code to the corresponding full name"""
  3355. return cls._country_map.get(code.upper())
  3356. class GeoUtils:
  3357. # Major IPv4 address blocks per country
  3358. _country_ip_map = {
  3359. 'AD': '46.172.224.0/19',
  3360. 'AE': '94.200.0.0/13',
  3361. 'AF': '149.54.0.0/17',
  3362. 'AG': '209.59.64.0/18',
  3363. 'AI': '204.14.248.0/21',
  3364. 'AL': '46.99.0.0/16',
  3365. 'AM': '46.70.0.0/15',
  3366. 'AO': '105.168.0.0/13',
  3367. 'AP': '182.50.184.0/21',
  3368. 'AQ': '23.154.160.0/24',
  3369. 'AR': '181.0.0.0/12',
  3370. 'AS': '202.70.112.0/20',
  3371. 'AT': '77.116.0.0/14',
  3372. 'AU': '1.128.0.0/11',
  3373. 'AW': '181.41.0.0/18',
  3374. 'AX': '185.217.4.0/22',
  3375. 'AZ': '5.197.0.0/16',
  3376. 'BA': '31.176.128.0/17',
  3377. 'BB': '65.48.128.0/17',
  3378. 'BD': '114.130.0.0/16',
  3379. 'BE': '57.0.0.0/8',
  3380. 'BF': '102.178.0.0/15',
  3381. 'BG': '95.42.0.0/15',
  3382. 'BH': '37.131.0.0/17',
  3383. 'BI': '154.117.192.0/18',
  3384. 'BJ': '137.255.0.0/16',
  3385. 'BL': '185.212.72.0/23',
  3386. 'BM': '196.12.64.0/18',
  3387. 'BN': '156.31.0.0/16',
  3388. 'BO': '161.56.0.0/16',
  3389. 'BQ': '161.0.80.0/20',
  3390. 'BR': '191.128.0.0/12',
  3391. 'BS': '24.51.64.0/18',
  3392. 'BT': '119.2.96.0/19',
  3393. 'BW': '168.167.0.0/16',
  3394. 'BY': '178.120.0.0/13',
  3395. 'BZ': '179.42.192.0/18',
  3396. 'CA': '99.224.0.0/11',
  3397. 'CD': '41.243.0.0/16',
  3398. 'CF': '197.242.176.0/21',
  3399. 'CG': '160.113.0.0/16',
  3400. 'CH': '85.0.0.0/13',
  3401. 'CI': '102.136.0.0/14',
  3402. 'CK': '202.65.32.0/19',
  3403. 'CL': '152.172.0.0/14',
  3404. 'CM': '102.244.0.0/14',
  3405. 'CN': '36.128.0.0/10',
  3406. 'CO': '181.240.0.0/12',
  3407. 'CR': '201.192.0.0/12',
  3408. 'CU': '152.206.0.0/15',
  3409. 'CV': '165.90.96.0/19',
  3410. 'CW': '190.88.128.0/17',
  3411. 'CY': '31.153.0.0/16',
  3412. 'CZ': '88.100.0.0/14',
  3413. 'DE': '53.0.0.0/8',
  3414. 'DJ': '197.241.0.0/17',
  3415. 'DK': '87.48.0.0/12',
  3416. 'DM': '192.243.48.0/20',
  3417. 'DO': '152.166.0.0/15',
  3418. 'DZ': '41.96.0.0/12',
  3419. 'EC': '186.68.0.0/15',
  3420. 'EE': '90.190.0.0/15',
  3421. 'EG': '156.160.0.0/11',
  3422. 'ER': '196.200.96.0/20',
  3423. 'ES': '88.0.0.0/11',
  3424. 'ET': '196.188.0.0/14',
  3425. 'EU': '2.16.0.0/13',
  3426. 'FI': '91.152.0.0/13',
  3427. 'FJ': '144.120.0.0/16',
  3428. 'FK': '80.73.208.0/21',
  3429. 'FM': '119.252.112.0/20',
  3430. 'FO': '88.85.32.0/19',
  3431. 'FR': '90.0.0.0/9',
  3432. 'GA': '41.158.0.0/15',
  3433. 'GB': '25.0.0.0/8',
  3434. 'GD': '74.122.88.0/21',
  3435. 'GE': '31.146.0.0/16',
  3436. 'GF': '161.22.64.0/18',
  3437. 'GG': '62.68.160.0/19',
  3438. 'GH': '154.160.0.0/12',
  3439. 'GI': '95.164.0.0/16',
  3440. 'GL': '88.83.0.0/19',
  3441. 'GM': '160.182.0.0/15',
  3442. 'GN': '197.149.192.0/18',
  3443. 'GP': '104.250.0.0/19',
  3444. 'GQ': '105.235.224.0/20',
  3445. 'GR': '94.64.0.0/13',
  3446. 'GT': '168.234.0.0/16',
  3447. 'GU': '168.123.0.0/16',
  3448. 'GW': '197.214.80.0/20',
  3449. 'GY': '181.41.64.0/18',
  3450. 'HK': '113.252.0.0/14',
  3451. 'HN': '181.210.0.0/16',
  3452. 'HR': '93.136.0.0/13',
  3453. 'HT': '148.102.128.0/17',
  3454. 'HU': '84.0.0.0/14',
  3455. 'ID': '39.192.0.0/10',
  3456. 'IE': '87.32.0.0/12',
  3457. 'IL': '79.176.0.0/13',
  3458. 'IM': '5.62.80.0/20',
  3459. 'IN': '117.192.0.0/10',
  3460. 'IO': '203.83.48.0/21',
  3461. 'IQ': '37.236.0.0/14',
  3462. 'IR': '2.176.0.0/12',
  3463. 'IS': '82.221.0.0/16',
  3464. 'IT': '79.0.0.0/10',
  3465. 'JE': '87.244.64.0/18',
  3466. 'JM': '72.27.0.0/17',
  3467. 'JO': '176.29.0.0/16',
  3468. 'JP': '133.0.0.0/8',
  3469. 'KE': '105.48.0.0/12',
  3470. 'KG': '158.181.128.0/17',
  3471. 'KH': '36.37.128.0/17',
  3472. 'KI': '103.25.140.0/22',
  3473. 'KM': '197.255.224.0/20',
  3474. 'KN': '198.167.192.0/19',
  3475. 'KP': '175.45.176.0/22',
  3476. 'KR': '175.192.0.0/10',
  3477. 'KW': '37.36.0.0/14',
  3478. 'KY': '64.96.0.0/15',
  3479. 'KZ': '2.72.0.0/13',
  3480. 'LA': '115.84.64.0/18',
  3481. 'LB': '178.135.0.0/16',
  3482. 'LC': '24.92.144.0/20',
  3483. 'LI': '82.117.0.0/19',
  3484. 'LK': '112.134.0.0/15',
  3485. 'LR': '102.183.0.0/16',
  3486. 'LS': '129.232.0.0/17',
  3487. 'LT': '78.56.0.0/13',
  3488. 'LU': '188.42.0.0/16',
  3489. 'LV': '46.109.0.0/16',
  3490. 'LY': '41.252.0.0/14',
  3491. 'MA': '105.128.0.0/11',
  3492. 'MC': '88.209.64.0/18',
  3493. 'MD': '37.246.0.0/16',
  3494. 'ME': '178.175.0.0/17',
  3495. 'MF': '74.112.232.0/21',
  3496. 'MG': '154.126.0.0/17',
  3497. 'MH': '117.103.88.0/21',
  3498. 'MK': '77.28.0.0/15',
  3499. 'ML': '154.118.128.0/18',
  3500. 'MM': '37.111.0.0/17',
  3501. 'MN': '49.0.128.0/17',
  3502. 'MO': '60.246.0.0/16',
  3503. 'MP': '202.88.64.0/20',
  3504. 'MQ': '109.203.224.0/19',
  3505. 'MR': '41.188.64.0/18',
  3506. 'MS': '208.90.112.0/22',
  3507. 'MT': '46.11.0.0/16',
  3508. 'MU': '105.16.0.0/12',
  3509. 'MV': '27.114.128.0/18',
  3510. 'MW': '102.70.0.0/15',
  3511. 'MX': '187.192.0.0/11',
  3512. 'MY': '175.136.0.0/13',
  3513. 'MZ': '197.218.0.0/15',
  3514. 'NA': '41.182.0.0/16',
  3515. 'NC': '101.101.0.0/18',
  3516. 'NE': '197.214.0.0/18',
  3517. 'NF': '203.17.240.0/22',
  3518. 'NG': '105.112.0.0/12',
  3519. 'NI': '186.76.0.0/15',
  3520. 'NL': '145.96.0.0/11',
  3521. 'NO': '84.208.0.0/13',
  3522. 'NP': '36.252.0.0/15',
  3523. 'NR': '203.98.224.0/19',
  3524. 'NU': '49.156.48.0/22',
  3525. 'NZ': '49.224.0.0/14',
  3526. 'OM': '5.36.0.0/15',
  3527. 'PA': '186.72.0.0/15',
  3528. 'PE': '186.160.0.0/14',
  3529. 'PF': '123.50.64.0/18',
  3530. 'PG': '124.240.192.0/19',
  3531. 'PH': '49.144.0.0/13',
  3532. 'PK': '39.32.0.0/11',
  3533. 'PL': '83.0.0.0/11',
  3534. 'PM': '70.36.0.0/20',
  3535. 'PR': '66.50.0.0/16',
  3536. 'PS': '188.161.0.0/16',
  3537. 'PT': '85.240.0.0/13',
  3538. 'PW': '202.124.224.0/20',
  3539. 'PY': '181.120.0.0/14',
  3540. 'QA': '37.210.0.0/15',
  3541. 'RE': '102.35.0.0/16',
  3542. 'RO': '79.112.0.0/13',
  3543. 'RS': '93.86.0.0/15',
  3544. 'RU': '5.136.0.0/13',
  3545. 'RW': '41.186.0.0/16',
  3546. 'SA': '188.48.0.0/13',
  3547. 'SB': '202.1.160.0/19',
  3548. 'SC': '154.192.0.0/11',
  3549. 'SD': '102.120.0.0/13',
  3550. 'SE': '78.64.0.0/12',
  3551. 'SG': '8.128.0.0/10',
  3552. 'SI': '188.196.0.0/14',
  3553. 'SK': '78.98.0.0/15',
  3554. 'SL': '102.143.0.0/17',
  3555. 'SM': '89.186.32.0/19',
  3556. 'SN': '41.82.0.0/15',
  3557. 'SO': '154.115.192.0/18',
  3558. 'SR': '186.179.128.0/17',
  3559. 'SS': '105.235.208.0/21',
  3560. 'ST': '197.159.160.0/19',
  3561. 'SV': '168.243.0.0/16',
  3562. 'SX': '190.102.0.0/20',
  3563. 'SY': '5.0.0.0/16',
  3564. 'SZ': '41.84.224.0/19',
  3565. 'TC': '65.255.48.0/20',
  3566. 'TD': '154.68.128.0/19',
  3567. 'TG': '196.168.0.0/14',
  3568. 'TH': '171.96.0.0/13',
  3569. 'TJ': '85.9.128.0/18',
  3570. 'TK': '27.96.24.0/21',
  3571. 'TL': '180.189.160.0/20',
  3572. 'TM': '95.85.96.0/19',
  3573. 'TN': '197.0.0.0/11',
  3574. 'TO': '175.176.144.0/21',
  3575. 'TR': '78.160.0.0/11',
  3576. 'TT': '186.44.0.0/15',
  3577. 'TV': '202.2.96.0/19',
  3578. 'TW': '120.96.0.0/11',
  3579. 'TZ': '156.156.0.0/14',
  3580. 'UA': '37.52.0.0/14',
  3581. 'UG': '102.80.0.0/13',
  3582. 'US': '6.0.0.0/8',
  3583. 'UY': '167.56.0.0/13',
  3584. 'UZ': '84.54.64.0/18',
  3585. 'VA': '212.77.0.0/19',
  3586. 'VC': '207.191.240.0/21',
  3587. 'VE': '186.88.0.0/13',
  3588. 'VG': '66.81.192.0/20',
  3589. 'VI': '146.226.0.0/16',
  3590. 'VN': '14.160.0.0/11',
  3591. 'VU': '202.80.32.0/20',
  3592. 'WF': '117.20.32.0/21',
  3593. 'WS': '202.4.32.0/19',
  3594. 'YE': '134.35.0.0/16',
  3595. 'YT': '41.242.116.0/22',
  3596. 'ZA': '41.0.0.0/11',
  3597. 'ZM': '102.144.0.0/13',
  3598. 'ZW': '102.177.192.0/18',
  3599. }
  3600. @classmethod
  3601. def random_ipv4(cls, code_or_block):
  3602. if len(code_or_block) == 2:
  3603. block = cls._country_ip_map.get(code_or_block.upper())
  3604. if not block:
  3605. return None
  3606. else:
  3607. block = code_or_block
  3608. addr, preflen = block.split('/')
  3609. addr_min = struct.unpack('!L', socket.inet_aton(addr))[0]
  3610. addr_max = addr_min | (0xffffffff >> int(preflen))
  3611. return str(socket.inet_ntoa(
  3612. struct.pack('!L', random.randint(addr_min, addr_max))))
  3613. # Both long_to_bytes and bytes_to_long are adapted from PyCrypto, which is
  3614. # released into Public Domain
  3615. # https://github.com/dlitz/pycrypto/blob/master/lib/Crypto/Util/number.py#L387
  3616. def long_to_bytes(n, blocksize=0):
  3617. """long_to_bytes(n:long, blocksize:int) : string
  3618. Convert a long integer to a byte string.
  3619. If optional blocksize is given and greater than zero, pad the front of the
  3620. byte string with binary zeros so that the length is a multiple of
  3621. blocksize.
  3622. """
  3623. # after much testing, this algorithm was deemed to be the fastest
  3624. s = b''
  3625. n = int(n)
  3626. while n > 0:
  3627. s = struct.pack('>I', n & 0xffffffff) + s
  3628. n = n >> 32
  3629. # strip off leading zeros
  3630. for i in range(len(s)):
  3631. if s[i] != b'\000'[0]:
  3632. break
  3633. else:
  3634. # only happens when n == 0
  3635. s = b'\000'
  3636. i = 0
  3637. s = s[i:]
  3638. # add back some pad bytes. this could be done more efficiently w.r.t. the
  3639. # de-padding being done above, but sigh...
  3640. if blocksize > 0 and len(s) % blocksize:
  3641. s = (blocksize - len(s) % blocksize) * b'\000' + s
  3642. return s
  3643. def bytes_to_long(s):
  3644. """bytes_to_long(string) : long
  3645. Convert a byte string to a long integer.
  3646. This is (essentially) the inverse of long_to_bytes().
  3647. """
  3648. acc = 0
  3649. length = len(s)
  3650. if length % 4:
  3651. extra = (4 - length % 4)
  3652. s = b'\000' * extra + s
  3653. length = length + extra
  3654. for i in range(0, length, 4):
  3655. acc = (acc << 32) + struct.unpack('>I', s[i:i + 4])[0]
  3656. return acc
  3657. def ohdave_rsa_encrypt(data, exponent, modulus):
  3658. """
  3659. Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/
  3660. Input:
  3661. data: data to encrypt, bytes-like object
  3662. exponent, modulus: parameter e and N of RSA algorithm, both integer
  3663. Output: hex string of encrypted data
  3664. Limitation: supports one block encryption only
  3665. """
  3666. payload = int(binascii.hexlify(data[::-1]), 16)
  3667. encrypted = pow(payload, exponent, modulus)
  3668. return f'{encrypted:x}'
  3669. def pkcs1pad(data, length):
  3670. """
  3671. Padding input data with PKCS#1 scheme
  3672. @param {int[]} data input data
  3673. @param {int} length target length
  3674. @returns {int[]} padded data
  3675. """
  3676. if len(data) > length - 11:
  3677. raise ValueError('Input data too long for PKCS#1 padding')
  3678. pseudo_random = [random.randint(0, 254) for _ in range(length - len(data) - 3)]
  3679. return [0, 2, *pseudo_random, 0, *data]
  3680. def _base_n_table(n, table):
  3681. if not table and not n:
  3682. raise ValueError('Either table or n must be specified')
  3683. table = (table or '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ')[:n]
  3684. if n and n != len(table):
  3685. raise ValueError(f'base {n} exceeds table length {len(table)}')
  3686. return table
  3687. def encode_base_n(num, n=None, table=None):
  3688. """Convert given int to a base-n string"""
  3689. table = _base_n_table(n, table)
  3690. if not num:
  3691. return table[0]
  3692. result, base = '', len(table)
  3693. while num:
  3694. result = table[num % base] + result
  3695. num = num // base
  3696. return result
  3697. def decode_base_n(string, n=None, table=None):
  3698. """Convert given base-n string to int"""
  3699. table = {char: index for index, char in enumerate(_base_n_table(n, table))}
  3700. result, base = 0, len(table)
  3701. for char in string:
  3702. result = result * base + table[char]
  3703. return result
  3704. def decode_packed_codes(code):
  3705. mobj = re.search(PACKED_CODES_RE, code)
  3706. obfuscated_code, base, count, symbols = mobj.groups()
  3707. base = int(base)
  3708. count = int(count)
  3709. symbols = symbols.split('|')
  3710. symbol_table = {}
  3711. while count:
  3712. count -= 1
  3713. base_n_count = encode_base_n(count, base)
  3714. symbol_table[base_n_count] = symbols[count] or base_n_count
  3715. return re.sub(
  3716. r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
  3717. obfuscated_code)
  3718. def caesar(s, alphabet, shift):
  3719. if shift == 0:
  3720. return s
  3721. l = len(alphabet)
  3722. return ''.join(
  3723. alphabet[(alphabet.index(c) + shift) % l] if c in alphabet else c
  3724. for c in s)
  3725. def rot47(s):
  3726. return caesar(s, r'''!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~''', 47)
  3727. def parse_m3u8_attributes(attrib):
  3728. info = {}
  3729. for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):
  3730. if val.startswith('"'):
  3731. val = val[1:-1]
  3732. info[key] = val
  3733. return info
  3734. def urshift(val, n):
  3735. return val >> n if val >= 0 else (val + 0x100000000) >> n
  3736. def write_xattr(path, key, value):
  3737. # Windows: Write xattrs to NTFS Alternate Data Streams:
  3738. # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
  3739. if compat_os_name == 'nt':
  3740. assert ':' not in key
  3741. assert os.path.exists(path)
  3742. try:
  3743. with open(f'{path}:{key}', 'wb') as f:
  3744. f.write(value)
  3745. except OSError as e:
  3746. raise XAttrMetadataError(e.errno, e.strerror)
  3747. return
  3748. # UNIX Method 1. Use os.setxattr/xattrs/pyxattrs modules
  3749. setxattr = None
  3750. if callable(getattr(os, 'setxattr', None)):
  3751. setxattr = os.setxattr
  3752. elif getattr(xattr, '_yt_dlp__identifier', None) == 'pyxattr':
  3753. # Unicode arguments are not supported in pyxattr until version 0.5.0
  3754. # See https://github.com/ytdl-org/youtube-dl/issues/5498
  3755. if version_tuple(xattr.__version__) >= (0, 5, 0):
  3756. setxattr = xattr.set
  3757. elif xattr:
  3758. setxattr = xattr.setxattr
  3759. if setxattr:
  3760. try:
  3761. setxattr(path, key, value)
  3762. except OSError as e:
  3763. raise XAttrMetadataError(e.errno, e.strerror)
  3764. return
  3765. # UNIX Method 2. Use setfattr/xattr executables
  3766. exe = ('setfattr' if check_executable('setfattr', ['--version'])
  3767. else 'xattr' if check_executable('xattr', ['-h']) else None)
  3768. if not exe:
  3769. raise XAttrUnavailableError(
  3770. 'Couldn\'t find a tool to set the xattrs. Install either the "xattr" or "pyxattr" Python modules or the '
  3771. + ('"xattr" binary' if sys.platform != 'linux' else 'GNU "attr" package (which contains the "setfattr" tool)'))
  3772. value = value.decode()
  3773. try:
  3774. _, stderr, returncode = Popen.run(
  3775. [exe, '-w', key, value, path] if exe == 'xattr' else [exe, '-n', key, '-v', value, path],
  3776. text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
  3777. except OSError as e:
  3778. raise XAttrMetadataError(e.errno, e.strerror)
  3779. if returncode:
  3780. raise XAttrMetadataError(returncode, stderr)
  3781. def random_birthday(year_field, month_field, day_field):
  3782. start_date = dt.date(1950, 1, 1)
  3783. end_date = dt.date(1995, 12, 31)
  3784. offset = random.randint(0, (end_date - start_date).days)
  3785. random_date = start_date + dt.timedelta(offset)
  3786. return {
  3787. year_field: str(random_date.year),
  3788. month_field: str(random_date.month),
  3789. day_field: str(random_date.day),
  3790. }
  3791. def find_available_port(interface=''):
  3792. try:
  3793. with socket.socket() as sock:
  3794. sock.bind((interface, 0))
  3795. return sock.getsockname()[1]
  3796. except OSError:
  3797. return None
  3798. # Templates for internet shortcut files, which are plain text files.
  3799. DOT_URL_LINK_TEMPLATE = '''\
  3800. [InternetShortcut]
  3801. URL=%(url)s
  3802. '''
  3803. DOT_WEBLOC_LINK_TEMPLATE = '''\
  3804. <?xml version="1.0" encoding="UTF-8"?>
  3805. <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
  3806. <plist version="1.0">
  3807. <dict>
  3808. \t<key>URL</key>
  3809. \t<string>%(url)s</string>
  3810. </dict>
  3811. </plist>
  3812. '''
  3813. DOT_DESKTOP_LINK_TEMPLATE = '''\
  3814. [Desktop Entry]
  3815. Encoding=UTF-8
  3816. Name=%(filename)s
  3817. Type=Link
  3818. URL=%(url)s
  3819. Icon=text-html
  3820. '''
  3821. LINK_TEMPLATES = {
  3822. 'url': DOT_URL_LINK_TEMPLATE,
  3823. 'desktop': DOT_DESKTOP_LINK_TEMPLATE,
  3824. 'webloc': DOT_WEBLOC_LINK_TEMPLATE,
  3825. }
  3826. def iri_to_uri(iri):
  3827. """
  3828. Converts an IRI (Internationalized Resource Identifier, allowing Unicode characters) to a URI (Uniform Resource Identifier, ASCII-only).
  3829. The function doesn't add an additional layer of escaping; e.g., it doesn't escape `%3C` as `%253C`. Instead, it percent-escapes characters with an underlying UTF-8 encoding *besides* those already escaped, leaving the URI intact.
  3830. """
  3831. iri_parts = urllib.parse.urlparse(iri)
  3832. if '[' in iri_parts.netloc:
  3833. raise ValueError('IPv6 URIs are not, yet, supported.')
  3834. # Querying `.netloc`, when there's only one bracket, also raises a ValueError.
  3835. # The `safe` argument values, that the following code uses, contain the characters that should not be percent-encoded. Everything else but letters, digits and '_.-' will be percent-encoded with an underlying UTF-8 encoding. Everything already percent-encoded will be left as is.
  3836. net_location = ''
  3837. if iri_parts.username:
  3838. net_location += urllib.parse.quote(iri_parts.username, safe=r"!$%&'()*+,~")
  3839. if iri_parts.password is not None:
  3840. net_location += ':' + urllib.parse.quote(iri_parts.password, safe=r"!$%&'()*+,~")
  3841. net_location += '@'
  3842. net_location += iri_parts.hostname.encode('idna').decode() # Punycode for Unicode hostnames.
  3843. # The 'idna' encoding produces ASCII text.
  3844. if iri_parts.port is not None and iri_parts.port != 80:
  3845. net_location += ':' + str(iri_parts.port)
  3846. return urllib.parse.urlunparse(
  3847. (iri_parts.scheme,
  3848. net_location,
  3849. urllib.parse.quote_plus(iri_parts.path, safe=r"!$%&'()*+,/:;=@|~"),
  3850. # Unsure about the `safe` argument, since this is a legacy way of handling parameters.
  3851. urllib.parse.quote_plus(iri_parts.params, safe=r"!$%&'()*+,/:;=@|~"),
  3852. # Not totally sure about the `safe` argument, since the source does not explicitly mention the query URI component.
  3853. urllib.parse.quote_plus(iri_parts.query, safe=r"!$%&'()*+,/:;=?@{|}~"),
  3854. urllib.parse.quote_plus(iri_parts.fragment, safe=r"!#$%&'()*+,/:;=?@{|}~")))
  3855. # Source for `safe` arguments: https://url.spec.whatwg.org/#percent-encoded-bytes.
  3856. def to_high_limit_path(path):
  3857. if sys.platform in ['win32', 'cygwin']:
  3858. # Work around MAX_PATH limitation on Windows. The maximum allowed length for the individual path segments may still be quite limited.
  3859. return '\\\\?\\' + os.path.abspath(path)
  3860. return path
  3861. def format_field(obj, field=None, template='%s', ignore=NO_DEFAULT, default='', func=IDENTITY):
  3862. val = traversal.traverse_obj(obj, *variadic(field))
  3863. if not val if ignore is NO_DEFAULT else val in variadic(ignore):
  3864. return default
  3865. return template % func(val)
  3866. def clean_podcast_url(url):
  3867. url = re.sub(r'''(?x)
  3868. (?:
  3869. (?:
  3870. chtbl\.com/track|
  3871. media\.blubrry\.com| # https://create.blubrry.com/resources/podcast-media-download-statistics/getting-started/
  3872. play\.podtrac\.com|
  3873. chrt\.fm/track|
  3874. mgln\.ai/e
  3875. )(?:/[^/.]+)?|
  3876. (?:dts|www)\.podtrac\.com/(?:pts/)?redirect\.[0-9a-z]{3,4}| # http://analytics.podtrac.com/how-to-measure
  3877. flex\.acast\.com|
  3878. pd(?:
  3879. cn\.co| # https://podcorn.com/analytics-prefix/
  3880. st\.fm # https://podsights.com/docs/
  3881. )/e|
  3882. [0-9]\.gum\.fm|
  3883. pscrb\.fm/rss/p
  3884. )/''', '', url)
  3885. return re.sub(r'^\w+://(\w+://)', r'\1', url)
  3886. _HEX_TABLE = '0123456789abcdef'
  3887. def random_uuidv4():
  3888. return re.sub(r'[xy]', lambda x: _HEX_TABLE[random.randint(0, 15)], 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx')
  3889. def make_dir(path, to_screen=None):
  3890. try:
  3891. dn = os.path.dirname(path)
  3892. if dn:
  3893. os.makedirs(dn, exist_ok=True)
  3894. return True
  3895. except OSError as err:
  3896. if callable(to_screen) is not None:
  3897. to_screen(f'unable to create directory {err}')
  3898. return False
  3899. def get_executable_path():
  3900. from ..update import _get_variant_and_executable_path
  3901. return os.path.dirname(os.path.abspath(_get_variant_and_executable_path()[1]))
  3902. def get_user_config_dirs(package_name):
  3903. # .config (e.g. ~/.config/package_name)
  3904. xdg_config_home = os.getenv('XDG_CONFIG_HOME') or compat_expanduser('~/.config')
  3905. yield os.path.join(xdg_config_home, package_name)
  3906. # appdata (%APPDATA%/package_name)
  3907. appdata_dir = os.getenv('appdata')
  3908. if appdata_dir:
  3909. yield os.path.join(appdata_dir, package_name)
  3910. # home (~/.package_name)
  3911. yield os.path.join(compat_expanduser('~'), f'.{package_name}')
  3912. def get_system_config_dirs(package_name):
  3913. # /etc/package_name
  3914. yield os.path.join('/etc', package_name)
  3915. def time_seconds(**kwargs):
  3916. """
  3917. Returns TZ-aware time in seconds since the epoch (1970-01-01T00:00:00Z)
  3918. """
  3919. return time.time() + dt.timedelta(**kwargs).total_seconds()
  3920. # create a JSON Web Signature (jws) with HS256 algorithm
  3921. # the resulting format is in JWS Compact Serialization
  3922. # implemented following JWT https://www.rfc-editor.org/rfc/rfc7519.html
  3923. # implemented following JWS https://www.rfc-editor.org/rfc/rfc7515.html
  3924. def jwt_encode_hs256(payload_data, key, headers={}):
  3925. header_data = {
  3926. 'alg': 'HS256',
  3927. 'typ': 'JWT',
  3928. }
  3929. if headers:
  3930. header_data.update(headers)
  3931. header_b64 = base64.b64encode(json.dumps(header_data).encode())
  3932. payload_b64 = base64.b64encode(json.dumps(payload_data).encode())
  3933. h = hmac.new(key.encode(), header_b64 + b'.' + payload_b64, hashlib.sha256)
  3934. signature_b64 = base64.b64encode(h.digest())
  3935. return header_b64 + b'.' + payload_b64 + b'.' + signature_b64
  3936. # can be extended in future to verify the signature and parse header and return the algorithm used if it's not HS256
  3937. def jwt_decode_hs256(jwt):
  3938. header_b64, payload_b64, signature_b64 = jwt.split('.')
  3939. # add trailing ='s that may have been stripped, superfluous ='s are ignored
  3940. return json.loads(base64.urlsafe_b64decode(f'{payload_b64}==='))
  3941. WINDOWS_VT_MODE = False if compat_os_name == 'nt' else None
  3942. @functools.cache
  3943. def supports_terminal_sequences(stream):
  3944. if compat_os_name == 'nt':
  3945. if not WINDOWS_VT_MODE:
  3946. return False
  3947. elif not os.getenv('TERM'):
  3948. return False
  3949. try:
  3950. return stream.isatty()
  3951. except BaseException:
  3952. return False
  3953. def windows_enable_vt_mode():
  3954. """Ref: https://bugs.python.org/issue30075 """
  3955. if get_windows_version() < (10, 0, 10586):
  3956. return
  3957. import ctypes
  3958. import ctypes.wintypes
  3959. import msvcrt
  3960. ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004
  3961. dll = ctypes.WinDLL('kernel32', use_last_error=False)
  3962. handle = os.open('CONOUT$', os.O_RDWR)
  3963. try:
  3964. h_out = ctypes.wintypes.HANDLE(msvcrt.get_osfhandle(handle))
  3965. dw_original_mode = ctypes.wintypes.DWORD()
  3966. success = dll.GetConsoleMode(h_out, ctypes.byref(dw_original_mode))
  3967. if not success:
  3968. raise Exception('GetConsoleMode failed')
  3969. success = dll.SetConsoleMode(h_out, ctypes.wintypes.DWORD(
  3970. dw_original_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING))
  3971. if not success:
  3972. raise Exception('SetConsoleMode failed')
  3973. finally:
  3974. os.close(handle)
  3975. global WINDOWS_VT_MODE
  3976. WINDOWS_VT_MODE = True
  3977. supports_terminal_sequences.cache_clear()
  3978. _terminal_sequences_re = re.compile('\033\\[[^m]+m')
  3979. def remove_terminal_sequences(string):
  3980. return _terminal_sequences_re.sub('', string)
  3981. def number_of_digits(number):
  3982. return len('%d' % number)
  3983. def join_nonempty(*values, delim='-', from_dict=None):
  3984. if from_dict is not None:
  3985. values = (traversal.traverse_obj(from_dict, variadic(v)) for v in values)
  3986. return delim.join(map(str, filter(None, values)))
  3987. def scale_thumbnails_to_max_format_width(formats, thumbnails, url_width_re):
  3988. """
  3989. Find the largest format dimensions in terms of video width and, for each thumbnail:
  3990. * Modify the URL: Match the width with the provided regex and replace with the former width
  3991. * Update dimensions
  3992. This function is useful with video services that scale the provided thumbnails on demand
  3993. """
  3994. _keys = ('width', 'height')
  3995. max_dimensions = max(
  3996. (tuple(fmt.get(k) or 0 for k in _keys) for fmt in formats),
  3997. default=(0, 0))
  3998. if not max_dimensions[0]:
  3999. return thumbnails
  4000. return [
  4001. merge_dicts(
  4002. {'url': re.sub(url_width_re, str(max_dimensions[0]), thumbnail['url'])},
  4003. dict(zip(_keys, max_dimensions)), thumbnail)
  4004. for thumbnail in thumbnails
  4005. ]
  4006. def parse_http_range(range):
  4007. """ Parse value of "Range" or "Content-Range" HTTP header into tuple. """
  4008. if not range:
  4009. return None, None, None
  4010. crg = re.search(r'bytes[ =](\d+)-(\d+)?(?:/(\d+))?', range)
  4011. if not crg:
  4012. return None, None, None
  4013. return int(crg.group(1)), int_or_none(crg.group(2)), int_or_none(crg.group(3))
  4014. def read_stdin(what):
  4015. if what:
  4016. eof = 'Ctrl+Z' if compat_os_name == 'nt' else 'Ctrl+D'
  4017. write_string(f'Reading {what} from STDIN - EOF ({eof}) to end:\n')
  4018. return sys.stdin
  4019. def determine_file_encoding(data):
  4020. """
  4021. Detect the text encoding used
  4022. @returns (encoding, bytes to skip)
  4023. """
  4024. # BOM marks are given priority over declarations
  4025. for bom, enc in BOMS:
  4026. if data.startswith(bom):
  4027. return enc, len(bom)
  4028. # Strip off all null bytes to match even when UTF-16 or UTF-32 is used.
  4029. # We ignore the endianness to get a good enough match
  4030. data = data.replace(b'\0', b'')
  4031. mobj = re.match(rb'(?m)^#\s*coding\s*:\s*(\S+)\s*$', data)
  4032. return mobj.group(1).decode() if mobj else None, 0
  4033. class Config:
  4034. own_args = None
  4035. parsed_args = None
  4036. filename = None
  4037. __initialized = False
  4038. def __init__(self, parser, label=None):
  4039. self.parser, self.label = parser, label
  4040. self._loaded_paths, self.configs = set(), []
  4041. def init(self, args=None, filename=None):
  4042. assert not self.__initialized
  4043. self.own_args, self.filename = args, filename
  4044. return self.load_configs()
  4045. def load_configs(self):
  4046. directory = ''
  4047. if self.filename:
  4048. location = os.path.realpath(self.filename)
  4049. directory = os.path.dirname(location)
  4050. if location in self._loaded_paths:
  4051. return False
  4052. self._loaded_paths.add(location)
  4053. self.__initialized = True
  4054. opts, _ = self.parser.parse_known_args(self.own_args)
  4055. self.parsed_args = self.own_args
  4056. for location in opts.config_locations or []:
  4057. if location == '-':
  4058. if location in self._loaded_paths:
  4059. continue
  4060. self._loaded_paths.add(location)
  4061. self.append_config(shlex.split(read_stdin('options'), comments=True), label='stdin')
  4062. continue
  4063. location = os.path.join(directory, expand_path(location))
  4064. if os.path.isdir(location):
  4065. location = os.path.join(location, 'yt-dlp.conf')
  4066. if not os.path.exists(location):
  4067. self.parser.error(f'config location {location} does not exist')
  4068. self.append_config(self.read_file(location), location)
  4069. return True
  4070. def __str__(self):
  4071. label = join_nonempty(
  4072. self.label, 'config', f'"{self.filename}"' if self.filename else '',
  4073. delim=' ')
  4074. return join_nonempty(
  4075. self.own_args is not None and f'{label[0].upper()}{label[1:]}: {self.hide_login_info(self.own_args)}',
  4076. *(f'\n{c}'.replace('\n', '\n| ')[1:] for c in self.configs),
  4077. delim='\n')
  4078. @staticmethod
  4079. def read_file(filename, default=[]):
  4080. try:
  4081. optionf = open(filename, 'rb')
  4082. except OSError:
  4083. return default # silently skip if file is not present
  4084. try:
  4085. enc, skip = determine_file_encoding(optionf.read(512))
  4086. optionf.seek(skip, io.SEEK_SET)
  4087. except OSError:
  4088. enc = None # silently skip read errors
  4089. try:
  4090. # FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
  4091. contents = optionf.read().decode(enc or preferredencoding())
  4092. res = shlex.split(contents, comments=True)
  4093. except Exception as err:
  4094. raise ValueError(f'Unable to parse "{filename}": {err}')
  4095. finally:
  4096. optionf.close()
  4097. return res
  4098. @staticmethod
  4099. def hide_login_info(opts):
  4100. PRIVATE_OPTS = {'-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username'}
  4101. eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')
  4102. def _scrub_eq(o):
  4103. m = eqre.match(o)
  4104. if m:
  4105. return m.group('key') + '=PRIVATE'
  4106. else:
  4107. return o
  4108. opts = list(map(_scrub_eq, opts))
  4109. for idx, opt in enumerate(opts):
  4110. if opt in PRIVATE_OPTS and idx + 1 < len(opts):
  4111. opts[idx + 1] = 'PRIVATE'
  4112. return opts
  4113. def append_config(self, *args, label=None):
  4114. config = type(self)(self.parser, label)
  4115. config._loaded_paths = self._loaded_paths
  4116. if config.init(*args):
  4117. self.configs.append(config)
  4118. @property
  4119. def all_args(self):
  4120. for config in reversed(self.configs):
  4121. yield from config.all_args
  4122. yield from self.parsed_args or []
  4123. def parse_known_args(self, **kwargs):
  4124. return self.parser.parse_known_args(self.all_args, **kwargs)
  4125. def parse_args(self):
  4126. return self.parser.parse_args(self.all_args)
  4127. def merge_headers(*dicts):
  4128. """Merge dicts of http headers case insensitively, prioritizing the latter ones"""
  4129. return {k.title(): v for k, v in itertools.chain.from_iterable(map(dict.items, dicts))}
  4130. def cached_method(f):
  4131. """Cache a method"""
  4132. signature = inspect.signature(f)
  4133. @functools.wraps(f)
  4134. def wrapper(self, *args, **kwargs):
  4135. bound_args = signature.bind(self, *args, **kwargs)
  4136. bound_args.apply_defaults()
  4137. key = tuple(bound_args.arguments.values())[1:]
  4138. cache = vars(self).setdefault('_cached_method__cache', {}).setdefault(f.__name__, {})
  4139. if key not in cache:
  4140. cache[key] = f(self, *args, **kwargs)
  4141. return cache[key]
  4142. return wrapper
  4143. class classproperty:
  4144. """property access for class methods with optional caching"""
  4145. def __new__(cls, func=None, *args, **kwargs):
  4146. if not func:
  4147. return functools.partial(cls, *args, **kwargs)
  4148. return super().__new__(cls)
  4149. def __init__(self, func, *, cache=False):
  4150. functools.update_wrapper(self, func)
  4151. self.func = func
  4152. self._cache = {} if cache else None
  4153. def __get__(self, _, cls):
  4154. if self._cache is None:
  4155. return self.func(cls)
  4156. elif cls not in self._cache:
  4157. self._cache[cls] = self.func(cls)
  4158. return self._cache[cls]
  4159. class function_with_repr:
  4160. def __init__(self, func, repr_=None):
  4161. functools.update_wrapper(self, func)
  4162. self.func, self.__repr = func, repr_
  4163. def __call__(self, *args, **kwargs):
  4164. return self.func(*args, **kwargs)
  4165. @classmethod
  4166. def set_repr(cls, repr_):
  4167. return functools.partial(cls, repr_=repr_)
  4168. def __repr__(self):
  4169. if self.__repr:
  4170. return self.__repr
  4171. return f'{self.func.__module__}.{self.func.__qualname__}'
  4172. class Namespace(types.SimpleNamespace):
  4173. """Immutable namespace"""
  4174. def __iter__(self):
  4175. return iter(self.__dict__.values())
  4176. @property
  4177. def items_(self):
  4178. return self.__dict__.items()
  4179. MEDIA_EXTENSIONS = Namespace(
  4180. common_video=('avi', 'flv', 'mkv', 'mov', 'mp4', 'webm'),
  4181. video=('3g2', '3gp', 'f4v', 'mk3d', 'divx', 'mpg', 'ogv', 'm4v', 'wmv'),
  4182. common_audio=('aiff', 'alac', 'flac', 'm4a', 'mka', 'mp3', 'ogg', 'opus', 'wav'),
  4183. audio=('aac', 'ape', 'asf', 'f4a', 'f4b', 'm4b', 'm4r', 'oga', 'ogx', 'spx', 'vorbis', 'wma', 'weba'),
  4184. thumbnails=('jpg', 'png', 'webp'),
  4185. storyboards=('mhtml', ),
  4186. subtitles=('srt', 'vtt', 'ass', 'lrc'),
  4187. manifests=('f4f', 'f4m', 'm3u8', 'smil', 'mpd'),
  4188. )
  4189. MEDIA_EXTENSIONS.video += MEDIA_EXTENSIONS.common_video
  4190. MEDIA_EXTENSIONS.audio += MEDIA_EXTENSIONS.common_audio
  4191. KNOWN_EXTENSIONS = (*MEDIA_EXTENSIONS.video, *MEDIA_EXTENSIONS.audio, *MEDIA_EXTENSIONS.manifests)
  4192. class _UnsafeExtensionError(Exception):
  4193. """
  4194. Mitigation exception for uncommon/malicious file extensions
  4195. This should be caught in YoutubeDL.py alongside a warning
  4196. Ref: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-79w7-vh3h-8g4j
  4197. """
  4198. ALLOWED_EXTENSIONS = frozenset([
  4199. # internal
  4200. 'description',
  4201. 'json',
  4202. 'meta',
  4203. 'orig',
  4204. 'part',
  4205. 'temp',
  4206. 'uncut',
  4207. 'unknown_video',
  4208. 'ytdl',
  4209. # video
  4210. *MEDIA_EXTENSIONS.video,
  4211. 'asx',
  4212. 'ismv',
  4213. 'm2t',
  4214. 'm2ts',
  4215. 'm2v',
  4216. 'm4s',
  4217. 'mng',
  4218. 'mp2v',
  4219. 'mp4v',
  4220. 'mpe',
  4221. 'mpeg',
  4222. 'mpeg1',
  4223. 'mpeg2',
  4224. 'mpeg4',
  4225. 'mxf',
  4226. 'ogm',
  4227. 'qt',
  4228. 'rm',
  4229. 'swf',
  4230. 'ts',
  4231. 'vob',
  4232. 'vp9',
  4233. # audio
  4234. *MEDIA_EXTENSIONS.audio,
  4235. '3ga',
  4236. 'ac3',
  4237. 'adts',
  4238. 'aif',
  4239. 'au',
  4240. 'dts',
  4241. 'isma',
  4242. 'it',
  4243. 'mid',
  4244. 'mod',
  4245. 'mpga',
  4246. 'mp1',
  4247. 'mp2',
  4248. 'mp4a',
  4249. 'mpa',
  4250. 'ra',
  4251. 'shn',
  4252. 'xm',
  4253. # image
  4254. *MEDIA_EXTENSIONS.thumbnails,
  4255. 'avif',
  4256. 'bmp',
  4257. 'gif',
  4258. 'heic',
  4259. 'ico',
  4260. 'image',
  4261. 'jng',
  4262. 'jpeg',
  4263. 'jxl',
  4264. 'svg',
  4265. 'tif',
  4266. 'tiff',
  4267. 'wbmp',
  4268. # subtitle
  4269. *MEDIA_EXTENSIONS.subtitles,
  4270. 'dfxp',
  4271. 'fs',
  4272. 'ismt',
  4273. 'json3',
  4274. 'sami',
  4275. 'scc',
  4276. 'srv1',
  4277. 'srv2',
  4278. 'srv3',
  4279. 'ssa',
  4280. 'tt',
  4281. 'ttml',
  4282. 'xml',
  4283. # others
  4284. *MEDIA_EXTENSIONS.manifests,
  4285. *MEDIA_EXTENSIONS.storyboards,
  4286. 'desktop',
  4287. 'ism',
  4288. 'm3u',
  4289. 'sbv',
  4290. 'url',
  4291. 'webloc',
  4292. ])
  4293. def __init__(self, extension, /):
  4294. super().__init__(f'unsafe file extension: {extension!r}')
  4295. self.extension = extension
  4296. @classmethod
  4297. def sanitize_extension(cls, extension, /, *, prepend=False):
  4298. if extension is None:
  4299. return None
  4300. if '/' in extension or '\\' in extension:
  4301. raise cls(extension)
  4302. if not prepend:
  4303. _, _, last = extension.rpartition('.')
  4304. if last == 'bin':
  4305. extension = last = 'unknown_video'
  4306. if last.lower() not in cls.ALLOWED_EXTENSIONS:
  4307. raise cls(extension)
  4308. return extension
  4309. class RetryManager:
  4310. """Usage:
  4311. for retry in RetryManager(...):
  4312. try:
  4313. ...
  4314. except SomeException as err:
  4315. retry.error = err
  4316. continue
  4317. """
  4318. attempt, _error = 0, None
  4319. def __init__(self, _retries, _error_callback, **kwargs):
  4320. self.retries = _retries or 0
  4321. self.error_callback = functools.partial(_error_callback, **kwargs)
  4322. def _should_retry(self):
  4323. return self._error is not NO_DEFAULT and self.attempt <= self.retries
  4324. @property
  4325. def error(self):
  4326. if self._error is NO_DEFAULT:
  4327. return None
  4328. return self._error
  4329. @error.setter
  4330. def error(self, value):
  4331. self._error = value
  4332. def __iter__(self):
  4333. while self._should_retry():
  4334. self.error = NO_DEFAULT
  4335. self.attempt += 1
  4336. yield self
  4337. if self.error:
  4338. self.error_callback(self.error, self.attempt, self.retries)
  4339. @staticmethod
  4340. def report_retry(e, count, retries, *, sleep_func, info, warn, error=None, suffix=None):
  4341. """Utility function for reporting retries"""
  4342. if count > retries:
  4343. if error:
  4344. return error(f'{e}. Giving up after {count - 1} retries') if count > 1 else error(str(e))
  4345. raise e
  4346. if not count:
  4347. return warn(e)
  4348. elif isinstance(e, ExtractorError):
  4349. e = remove_end(str_or_none(e.cause) or e.orig_msg, '.')
  4350. warn(f'{e}. Retrying{format_field(suffix, None, " %s")} ({count}/{retries})...')
  4351. delay = float_or_none(sleep_func(n=count - 1)) if callable(sleep_func) else sleep_func
  4352. if delay:
  4353. info(f'Sleeping {delay:.2f} seconds ...')
  4354. time.sleep(delay)
  4355. def make_archive_id(ie, video_id):
  4356. ie_key = ie if isinstance(ie, str) else ie.ie_key()
  4357. return f'{ie_key.lower()} {video_id}'
  4358. def truncate_string(s, left, right=0):
  4359. assert left > 3 and right >= 0
  4360. if s is None or len(s) <= left + right:
  4361. return s
  4362. return f'{s[:left - 3]}...{s[-right:] if right else ""}'
  4363. def orderedSet_from_options(options, alias_dict, *, use_regex=False, start=None):
  4364. assert 'all' in alias_dict, '"all" alias is required'
  4365. requested = list(start or [])
  4366. for val in options:
  4367. discard = val.startswith('-')
  4368. if discard:
  4369. val = val[1:]
  4370. if val in alias_dict:
  4371. val = alias_dict[val] if not discard else [
  4372. i[1:] if i.startswith('-') else f'-{i}' for i in alias_dict[val]]
  4373. # NB: Do not allow regex in aliases for performance
  4374. requested = orderedSet_from_options(val, alias_dict, start=requested)
  4375. continue
  4376. current = (filter(re.compile(val, re.I).fullmatch, alias_dict['all']) if use_regex
  4377. else [val] if val in alias_dict['all'] else None)
  4378. if current is None:
  4379. raise ValueError(val)
  4380. if discard:
  4381. for item in current:
  4382. while item in requested:
  4383. requested.remove(item)
  4384. else:
  4385. requested.extend(current)
  4386. return orderedSet(requested)
  4387. # TODO: Rewrite
  4388. class FormatSorter:
  4389. regex = r' *((?P<reverse>\+)?(?P<field>[a-zA-Z0-9_]+)((?P<separator>[~:])(?P<limit>.*?))?)? *$'
  4390. default = ('hidden', 'aud_or_vid', 'hasvid', 'ie_pref', 'lang', 'quality',
  4391. 'res', 'fps', 'hdr:12', 'vcodec:vp9.2', 'channels', 'acodec',
  4392. 'size', 'br', 'asr', 'proto', 'ext', 'hasaud', 'source', 'id') # These must not be aliases
  4393. ytdl_default = ('hasaud', 'lang', 'quality', 'tbr', 'filesize', 'vbr',
  4394. 'height', 'width', 'proto', 'vext', 'abr', 'aext',
  4395. 'fps', 'fs_approx', 'source', 'id')
  4396. settings = {
  4397. 'vcodec': {'type': 'ordered', 'regex': True,
  4398. 'order': ['av0?1', 'vp0?9.0?2', 'vp0?9', '[hx]265|he?vc?', '[hx]264|avc', 'vp0?8', 'mp4v|h263', 'theora', '', None, 'none']},
  4399. 'acodec': {'type': 'ordered', 'regex': True,
  4400. 'order': ['[af]lac', 'wav|aiff', 'opus', 'vorbis|ogg', 'aac', 'mp?4a?', 'mp3', 'ac-?4', 'e-?a?c-?3', 'ac-?3', 'dts', '', None, 'none']},
  4401. 'hdr': {'type': 'ordered', 'regex': True, 'field': 'dynamic_range',
  4402. 'order': ['dv', '(hdr)?12', r'(hdr)?10\+', '(hdr)?10', 'hlg', '', 'sdr', None]},
  4403. 'proto': {'type': 'ordered', 'regex': True, 'field': 'protocol',
  4404. 'order': ['(ht|f)tps', '(ht|f)tp$', 'm3u8.*', '.*dash', 'websocket_frag', 'rtmpe?', '', 'mms|rtsp', 'ws|websocket', 'f4']},
  4405. 'vext': {'type': 'ordered', 'field': 'video_ext',
  4406. 'order': ('mp4', 'mov', 'webm', 'flv', '', 'none'),
  4407. 'order_free': ('webm', 'mp4', 'mov', 'flv', '', 'none')},
  4408. 'aext': {'type': 'ordered', 'regex': True, 'field': 'audio_ext',
  4409. 'order': ('m4a', 'aac', 'mp3', 'ogg', 'opus', 'web[am]', '', 'none'),
  4410. 'order_free': ('ogg', 'opus', 'web[am]', 'mp3', 'm4a', 'aac', '', 'none')},
  4411. 'hidden': {'visible': False, 'forced': True, 'type': 'extractor', 'max': -1000},
  4412. 'aud_or_vid': {'visible': False, 'forced': True, 'type': 'multiple',
  4413. 'field': ('vcodec', 'acodec'),
  4414. 'function': lambda it: int(any(v != 'none' for v in it))},
  4415. 'ie_pref': {'priority': True, 'type': 'extractor'},
  4416. 'hasvid': {'priority': True, 'field': 'vcodec', 'type': 'boolean', 'not_in_list': ('none',)},
  4417. 'hasaud': {'field': 'acodec', 'type': 'boolean', 'not_in_list': ('none',)},
  4418. 'lang': {'convert': 'float', 'field': 'language_preference', 'default': -1},
  4419. 'quality': {'convert': 'float', 'default': -1},
  4420. 'filesize': {'convert': 'bytes'},
  4421. 'fs_approx': {'convert': 'bytes', 'field': 'filesize_approx'},
  4422. 'id': {'convert': 'string', 'field': 'format_id'},
  4423. 'height': {'convert': 'float_none'},
  4424. 'width': {'convert': 'float_none'},
  4425. 'fps': {'convert': 'float_none'},
  4426. 'channels': {'convert': 'float_none', 'field': 'audio_channels'},
  4427. 'tbr': {'convert': 'float_none'},
  4428. 'vbr': {'convert': 'float_none'},
  4429. 'abr': {'convert': 'float_none'},
  4430. 'asr': {'convert': 'float_none'},
  4431. 'source': {'convert': 'float', 'field': 'source_preference', 'default': -1},
  4432. 'codec': {'type': 'combined', 'field': ('vcodec', 'acodec')},
  4433. 'br': {'type': 'multiple', 'field': ('tbr', 'vbr', 'abr'), 'convert': 'float_none',
  4434. 'function': lambda it: next(filter(None, it), None)},
  4435. 'size': {'type': 'multiple', 'field': ('filesize', 'fs_approx'), 'convert': 'bytes',
  4436. 'function': lambda it: next(filter(None, it), None)},
  4437. 'ext': {'type': 'combined', 'field': ('vext', 'aext')},
  4438. 'res': {'type': 'multiple', 'field': ('height', 'width'),
  4439. 'function': lambda it: min(filter(None, it), default=0)},
  4440. # Actual field names
  4441. 'format_id': {'type': 'alias', 'field': 'id'},
  4442. 'preference': {'type': 'alias', 'field': 'ie_pref'},
  4443. 'language_preference': {'type': 'alias', 'field': 'lang'},
  4444. 'source_preference': {'type': 'alias', 'field': 'source'},
  4445. 'protocol': {'type': 'alias', 'field': 'proto'},
  4446. 'filesize_approx': {'type': 'alias', 'field': 'fs_approx'},
  4447. 'audio_channels': {'type': 'alias', 'field': 'channels'},
  4448. # Deprecated
  4449. 'dimension': {'type': 'alias', 'field': 'res', 'deprecated': True},
  4450. 'resolution': {'type': 'alias', 'field': 'res', 'deprecated': True},
  4451. 'extension': {'type': 'alias', 'field': 'ext', 'deprecated': True},
  4452. 'bitrate': {'type': 'alias', 'field': 'br', 'deprecated': True},
  4453. 'total_bitrate': {'type': 'alias', 'field': 'tbr', 'deprecated': True},
  4454. 'video_bitrate': {'type': 'alias', 'field': 'vbr', 'deprecated': True},
  4455. 'audio_bitrate': {'type': 'alias', 'field': 'abr', 'deprecated': True},
  4456. 'framerate': {'type': 'alias', 'field': 'fps', 'deprecated': True},
  4457. 'filesize_estimate': {'type': 'alias', 'field': 'size', 'deprecated': True},
  4458. 'samplerate': {'type': 'alias', 'field': 'asr', 'deprecated': True},
  4459. 'video_ext': {'type': 'alias', 'field': 'vext', 'deprecated': True},
  4460. 'audio_ext': {'type': 'alias', 'field': 'aext', 'deprecated': True},
  4461. 'video_codec': {'type': 'alias', 'field': 'vcodec', 'deprecated': True},
  4462. 'audio_codec': {'type': 'alias', 'field': 'acodec', 'deprecated': True},
  4463. 'video': {'type': 'alias', 'field': 'hasvid', 'deprecated': True},
  4464. 'has_video': {'type': 'alias', 'field': 'hasvid', 'deprecated': True},
  4465. 'audio': {'type': 'alias', 'field': 'hasaud', 'deprecated': True},
  4466. 'has_audio': {'type': 'alias', 'field': 'hasaud', 'deprecated': True},
  4467. 'extractor': {'type': 'alias', 'field': 'ie_pref', 'deprecated': True},
  4468. 'extractor_preference': {'type': 'alias', 'field': 'ie_pref', 'deprecated': True},
  4469. }
  4470. def __init__(self, ydl, field_preference):
  4471. self.ydl = ydl
  4472. self._order = []
  4473. self.evaluate_params(self.ydl.params, field_preference)
  4474. if ydl.params.get('verbose'):
  4475. self.print_verbose_info(self.ydl.write_debug)
  4476. def _get_field_setting(self, field, key):
  4477. if field not in self.settings:
  4478. if key in ('forced', 'priority'):
  4479. return False
  4480. self.ydl.deprecated_feature(f'Using arbitrary fields ({field}) for format sorting is '
  4481. 'deprecated and may be removed in a future version')
  4482. self.settings[field] = {}
  4483. prop_obj = self.settings[field]
  4484. if key not in prop_obj:
  4485. type_ = prop_obj.get('type')
  4486. if key == 'field':
  4487. default = 'preference' if type_ == 'extractor' else (field,) if type_ in ('combined', 'multiple') else field
  4488. elif key == 'convert':
  4489. default = 'order' if type_ == 'ordered' else 'float_string' if field else 'ignore'
  4490. else:
  4491. default = {'type': 'field', 'visible': True, 'order': [], 'not_in_list': (None,)}.get(key)
  4492. prop_obj[key] = default
  4493. return prop_obj[key]
  4494. def _resolve_field_value(self, field, value, convert_none=False):
  4495. if value is None:
  4496. if not convert_none:
  4497. return None
  4498. else:
  4499. value = value.lower()
  4500. conversion = self._get_field_setting(field, 'convert')
  4501. if conversion == 'ignore':
  4502. return None
  4503. if conversion == 'string':
  4504. return value
  4505. elif conversion == 'float_none':
  4506. return float_or_none(value)
  4507. elif conversion == 'bytes':
  4508. return parse_bytes(value)
  4509. elif conversion == 'order':
  4510. order_list = (self._use_free_order and self._get_field_setting(field, 'order_free')) or self._get_field_setting(field, 'order')
  4511. use_regex = self._get_field_setting(field, 'regex')
  4512. list_length = len(order_list)
  4513. empty_pos = order_list.index('') if '' in order_list else list_length + 1
  4514. if use_regex and value is not None:
  4515. for i, regex in enumerate(order_list):
  4516. if regex and re.match(regex, value):
  4517. return list_length - i
  4518. return list_length - empty_pos # not in list
  4519. else: # not regex or value = None
  4520. return list_length - (order_list.index(value) if value in order_list else empty_pos)
  4521. else:
  4522. if value.isnumeric():
  4523. return float(value)
  4524. else:
  4525. self.settings[field]['convert'] = 'string'
  4526. return value
  4527. def evaluate_params(self, params, sort_extractor):
  4528. self._use_free_order = params.get('prefer_free_formats', False)
  4529. self._sort_user = params.get('format_sort', [])
  4530. self._sort_extractor = sort_extractor
  4531. def add_item(field, reverse, closest, limit_text):
  4532. field = field.lower()
  4533. if field in self._order:
  4534. return
  4535. self._order.append(field)
  4536. limit = self._resolve_field_value(field, limit_text)
  4537. data = {
  4538. 'reverse': reverse,
  4539. 'closest': False if limit is None else closest,
  4540. 'limit_text': limit_text,
  4541. 'limit': limit}
  4542. if field in self.settings:
  4543. self.settings[field].update(data)
  4544. else:
  4545. self.settings[field] = data
  4546. sort_list = (
  4547. tuple(field for field in self.default if self._get_field_setting(field, 'forced'))
  4548. + (tuple() if params.get('format_sort_force', False)
  4549. else tuple(field for field in self.default if self._get_field_setting(field, 'priority')))
  4550. + tuple(self._sort_user) + tuple(sort_extractor) + self.default)
  4551. for item in sort_list:
  4552. match = re.match(self.regex, item)
  4553. if match is None:
  4554. raise ExtractorError(f'Invalid format sort string "{item}" given by extractor')
  4555. field = match.group('field')
  4556. if field is None:
  4557. continue
  4558. if self._get_field_setting(field, 'type') == 'alias':
  4559. alias, field = field, self._get_field_setting(field, 'field')
  4560. if self._get_field_setting(alias, 'deprecated'):
  4561. self.ydl.deprecated_feature(f'Format sorting alias {alias} is deprecated and may '
  4562. f'be removed in a future version. Please use {field} instead')
  4563. reverse = match.group('reverse') is not None
  4564. closest = match.group('separator') == '~'
  4565. limit_text = match.group('limit')
  4566. has_limit = limit_text is not None
  4567. has_multiple_fields = self._get_field_setting(field, 'type') == 'combined'
  4568. has_multiple_limits = has_limit and has_multiple_fields and not self._get_field_setting(field, 'same_limit')
  4569. fields = self._get_field_setting(field, 'field') if has_multiple_fields else (field,)
  4570. limits = limit_text.split(':') if has_multiple_limits else (limit_text,) if has_limit else tuple()
  4571. limit_count = len(limits)
  4572. for (i, f) in enumerate(fields):
  4573. add_item(f, reverse, closest,
  4574. limits[i] if i < limit_count
  4575. else limits[0] if has_limit and not has_multiple_limits
  4576. else None)
  4577. def print_verbose_info(self, write_debug):
  4578. if self._sort_user:
  4579. write_debug('Sort order given by user: {}'.format(', '.join(self._sort_user)))
  4580. if self._sort_extractor:
  4581. write_debug('Sort order given by extractor: {}'.format(', '.join(self._sort_extractor)))
  4582. write_debug('Formats sorted by: {}'.format(', '.join(['{}{}{}'.format(
  4583. '+' if self._get_field_setting(field, 'reverse') else '', field,
  4584. '{}{}({})'.format('~' if self._get_field_setting(field, 'closest') else ':',
  4585. self._get_field_setting(field, 'limit_text'),
  4586. self._get_field_setting(field, 'limit'))
  4587. if self._get_field_setting(field, 'limit_text') is not None else '')
  4588. for field in self._order if self._get_field_setting(field, 'visible')])))
  4589. def _calculate_field_preference_from_value(self, format_, field, type_, value):
  4590. reverse = self._get_field_setting(field, 'reverse')
  4591. closest = self._get_field_setting(field, 'closest')
  4592. limit = self._get_field_setting(field, 'limit')
  4593. if type_ == 'extractor':
  4594. maximum = self._get_field_setting(field, 'max')
  4595. if value is None or (maximum is not None and value >= maximum):
  4596. value = -1
  4597. elif type_ == 'boolean':
  4598. in_list = self._get_field_setting(field, 'in_list')
  4599. not_in_list = self._get_field_setting(field, 'not_in_list')
  4600. value = 0 if ((in_list is None or value in in_list) and (not_in_list is None or value not in not_in_list)) else -1
  4601. elif type_ == 'ordered':
  4602. value = self._resolve_field_value(field, value, True)
  4603. # try to convert to number
  4604. val_num = float_or_none(value, default=self._get_field_setting(field, 'default'))
  4605. is_num = self._get_field_setting(field, 'convert') != 'string' and val_num is not None
  4606. if is_num:
  4607. value = val_num
  4608. return ((-10, 0) if value is None
  4609. else (1, value, 0) if not is_num # if a field has mixed strings and numbers, strings are sorted higher
  4610. else (0, -abs(value - limit), value - limit if reverse else limit - value) if closest
  4611. else (0, value, 0) if not reverse and (limit is None or value <= limit)
  4612. else (0, -value, 0) if limit is None or (reverse and value == limit) or value > limit
  4613. else (-1, value, 0))
  4614. def _calculate_field_preference(self, format_, field):
  4615. type_ = self._get_field_setting(field, 'type') # extractor, boolean, ordered, field, multiple
  4616. get_value = lambda f: format_.get(self._get_field_setting(f, 'field'))
  4617. if type_ == 'multiple':
  4618. type_ = 'field' # Only 'field' is allowed in multiple for now
  4619. actual_fields = self._get_field_setting(field, 'field')
  4620. value = self._get_field_setting(field, 'function')(get_value(f) for f in actual_fields)
  4621. else:
  4622. value = get_value(field)
  4623. return self._calculate_field_preference_from_value(format_, field, type_, value)
  4624. def calculate_preference(self, format):
  4625. # Determine missing protocol
  4626. if not format.get('protocol'):
  4627. format['protocol'] = determine_protocol(format)
  4628. # Determine missing ext
  4629. if not format.get('ext') and 'url' in format:
  4630. format['ext'] = determine_ext(format['url'])
  4631. if format.get('vcodec') == 'none':
  4632. format['audio_ext'] = format['ext'] if format.get('acodec') != 'none' else 'none'
  4633. format['video_ext'] = 'none'
  4634. else:
  4635. format['video_ext'] = format['ext']
  4636. format['audio_ext'] = 'none'
  4637. # if format.get('preference') is None and format.get('ext') in ('f4f', 'f4m'): # Not supported?
  4638. # format['preference'] = -1000
  4639. if format.get('preference') is None and format.get('ext') == 'flv' and re.match('[hx]265|he?vc?', format.get('vcodec') or ''):
  4640. # HEVC-over-FLV is out-of-spec by FLV's original spec
  4641. # ref. https://trac.ffmpeg.org/ticket/6389
  4642. # ref. https://github.com/yt-dlp/yt-dlp/pull/5821
  4643. format['preference'] = -100
  4644. # Determine missing bitrates
  4645. if format.get('vcodec') == 'none':
  4646. format['vbr'] = 0
  4647. if format.get('acodec') == 'none':
  4648. format['abr'] = 0
  4649. if not format.get('vbr') and format.get('vcodec') != 'none':
  4650. format['vbr'] = try_call(lambda: format['tbr'] - format['abr']) or None
  4651. if not format.get('abr') and format.get('acodec') != 'none':
  4652. format['abr'] = try_call(lambda: format['tbr'] - format['vbr']) or None
  4653. if not format.get('tbr'):
  4654. format['tbr'] = try_call(lambda: format['vbr'] + format['abr']) or None
  4655. return tuple(self._calculate_field_preference(format, field) for field in self._order)
  4656. def filesize_from_tbr(tbr, duration):
  4657. """
  4658. @param tbr: Total bitrate in kbps (1000 bits/sec)
  4659. @param duration: Duration in seconds
  4660. @returns Filesize in bytes
  4661. """
  4662. if tbr is None or duration is None:
  4663. return None
  4664. return int(duration * tbr * (1000 / 8))
  4665. # XXX: Temporary
  4666. class _YDLLogger:
  4667. def __init__(self, ydl=None):
  4668. self._ydl = ydl
  4669. def debug(self, message):
  4670. if self._ydl:
  4671. self._ydl.write_debug(message)
  4672. def info(self, message):
  4673. if self._ydl:
  4674. self._ydl.to_screen(message)
  4675. def warning(self, message, *, once=False):
  4676. if self._ydl:
  4677. self._ydl.report_warning(message, once)
  4678. def error(self, message, *, is_error=True):
  4679. if self._ydl:
  4680. self._ydl.report_error(message, is_error=is_error)
  4681. def stdout(self, message):
  4682. if self._ydl:
  4683. self._ydl.to_stdout(message)
  4684. def stderr(self, message):
  4685. if self._ydl:
  4686. self._ydl.to_stderr(message)