test_subtitles.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451
  1. #!/usr/bin/env python3
  2. # Allow direct execution
  3. import os
  4. import sys
  5. import unittest
  6. sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  7. from test.helper import FakeYDL, is_download_test, md5
  8. from yt_dlp.extractor import (
  9. NPOIE,
  10. NRKTVIE,
  11. PBSIE,
  12. CeskaTelevizeIE,
  13. ComedyCentralIE,
  14. DailymotionIE,
  15. DemocracynowIE,
  16. LyndaIE,
  17. RaiPlayIE,
  18. RTVEALaCartaIE,
  19. TedTalkIE,
  20. ThePlatformFeedIE,
  21. ThePlatformIE,
  22. VikiIE,
  23. VimeoIE,
  24. WallaIE,
  25. YoutubeIE,
  26. )
  27. @is_download_test
  28. class BaseTestSubtitles(unittest.TestCase):
  29. url = None
  30. IE = None
  31. def setUp(self):
  32. self.DL = FakeYDL()
  33. self.ie = self.IE()
  34. self.DL.add_info_extractor(self.ie)
  35. if not self.IE.working():
  36. print(f'Skipping: {self.IE.ie_key()} marked as not _WORKING')
  37. self.skipTest('IE marked as not _WORKING')
  38. def getInfoDict(self):
  39. return self.DL.extract_info(self.url, download=False)
  40. def getSubtitles(self):
  41. info_dict = self.getInfoDict()
  42. subtitles = info_dict['requested_subtitles']
  43. if not subtitles:
  44. return subtitles
  45. for sub_info in subtitles.values():
  46. if sub_info.get('data') is None:
  47. uf = self.DL.urlopen(sub_info['url'])
  48. sub_info['data'] = uf.read().decode()
  49. return {l: sub_info['data'] for l, sub_info in subtitles.items()}
  50. @is_download_test
  51. class TestYoutubeSubtitles(BaseTestSubtitles):
  52. # Available subtitles for QRS8MkLhQmM:
  53. # Language formats
  54. # ru vtt, ttml, srv3, srv2, srv1, json3
  55. # fr vtt, ttml, srv3, srv2, srv1, json3
  56. # en vtt, ttml, srv3, srv2, srv1, json3
  57. # nl vtt, ttml, srv3, srv2, srv1, json3
  58. # de vtt, ttml, srv3, srv2, srv1, json3
  59. # ko vtt, ttml, srv3, srv2, srv1, json3
  60. # it vtt, ttml, srv3, srv2, srv1, json3
  61. # zh-Hant vtt, ttml, srv3, srv2, srv1, json3
  62. # hi vtt, ttml, srv3, srv2, srv1, json3
  63. # pt-BR vtt, ttml, srv3, srv2, srv1, json3
  64. # es-MX vtt, ttml, srv3, srv2, srv1, json3
  65. # ja vtt, ttml, srv3, srv2, srv1, json3
  66. # pl vtt, ttml, srv3, srv2, srv1, json3
  67. url = 'QRS8MkLhQmM'
  68. IE = YoutubeIE
  69. def test_youtube_allsubtitles(self):
  70. self.DL.params['writesubtitles'] = True
  71. self.DL.params['allsubtitles'] = True
  72. subtitles = self.getSubtitles()
  73. self.assertEqual(len(subtitles.keys()), 13)
  74. self.assertEqual(md5(subtitles['en']), 'ae1bd34126571a77aabd4d276b28044d')
  75. self.assertEqual(md5(subtitles['it']), '0e0b667ba68411d88fd1c5f4f4eab2f9')
  76. for lang in ['fr', 'de']:
  77. self.assertTrue(subtitles.get(lang) is not None, f'Subtitles for \'{lang}\' not extracted')
  78. def _test_subtitles_format(self, fmt, md5_hash, lang='en'):
  79. self.DL.params['writesubtitles'] = True
  80. self.DL.params['subtitlesformat'] = fmt
  81. subtitles = self.getSubtitles()
  82. self.assertEqual(md5(subtitles[lang]), md5_hash)
  83. def test_youtube_subtitles_ttml_format(self):
  84. self._test_subtitles_format('ttml', 'c97ddf1217390906fa9fbd34901f3da2')
  85. def test_youtube_subtitles_vtt_format(self):
  86. self._test_subtitles_format('vtt', 'ae1bd34126571a77aabd4d276b28044d')
  87. def test_youtube_subtitles_json3_format(self):
  88. self._test_subtitles_format('json3', '688dd1ce0981683867e7fe6fde2a224b')
  89. def _test_automatic_captions(self, url, lang):
  90. self.url = url
  91. self.DL.params['writeautomaticsub'] = True
  92. self.DL.params['subtitleslangs'] = [lang]
  93. subtitles = self.getSubtitles()
  94. self.assertTrue(subtitles[lang] is not None)
  95. def test_youtube_automatic_captions(self):
  96. # Available automatic captions for 8YoUxe5ncPo:
  97. # Language formats (all in vtt, ttml, srv3, srv2, srv1, json3)
  98. # gu, zh-Hans, zh-Hant, gd, ga, gl, lb, la, lo, tt, tr,
  99. # lv, lt, tk, th, tg, te, fil, haw, yi, ceb, yo, de, da,
  100. # el, eo, en, eu, et, es, ru, rw, ro, bn, be, bg, uk, jv,
  101. # bs, ja, or, xh, co, ca, cy, cs, ps, pt, pa, vi, pl, hy,
  102. # hr, ht, hu, hmn, hi, ha, mg, uz, ml, mn, mi, mk, ur,
  103. # mt, ms, mr, ug, ta, my, af, sw, is, am,
  104. # *it*, iw, sv, ar,
  105. # su, zu, az, id, ig, nl, no, ne, ny, fr, ku, fy, fa, fi,
  106. # ka, kk, sr, sq, ko, kn, km, st, sk, si, so, sn, sm, sl,
  107. # ky, sd
  108. # ...
  109. self._test_automatic_captions('8YoUxe5ncPo', 'it')
  110. @unittest.skip('Video unavailable')
  111. def test_youtube_translated_subtitles(self):
  112. # This video has a subtitles track, which can be translated (#4555)
  113. self._test_automatic_captions('Ky9eprVWzlI', 'it')
  114. def test_youtube_nosubtitles(self):
  115. self.DL.expect_warning('video doesn\'t have subtitles')
  116. # Available automatic captions for 8YoUxe5ncPo:
  117. # ...
  118. # 8YoUxe5ncPo has no subtitles
  119. self.url = '8YoUxe5ncPo'
  120. self.DL.params['writesubtitles'] = True
  121. self.DL.params['allsubtitles'] = True
  122. subtitles = self.getSubtitles()
  123. self.assertFalse(subtitles)
  124. @is_download_test
  125. class TestDailymotionSubtitles(BaseTestSubtitles):
  126. url = 'http://www.dailymotion.com/video/xczg00'
  127. IE = DailymotionIE
  128. def test_allsubtitles(self):
  129. self.DL.params['writesubtitles'] = True
  130. self.DL.params['allsubtitles'] = True
  131. subtitles = self.getSubtitles()
  132. self.assertTrue(len(subtitles.keys()) >= 6)
  133. self.assertEqual(md5(subtitles['en']), '976553874490cba125086bbfea3ff76f')
  134. self.assertEqual(md5(subtitles['fr']), '594564ec7d588942e384e920e5341792')
  135. for lang in ['es', 'fr', 'de']:
  136. self.assertTrue(subtitles.get(lang) is not None, f'Subtitles for \'{lang}\' not extracted')
  137. def test_nosubtitles(self):
  138. self.DL.expect_warning('video doesn\'t have subtitles')
  139. self.url = 'http://www.dailymotion.com/video/x12u166_le-zapping-tele-star-du-08-aout-2013_tv'
  140. self.DL.params['writesubtitles'] = True
  141. self.DL.params['allsubtitles'] = True
  142. subtitles = self.getSubtitles()
  143. self.assertFalse(subtitles)
  144. @is_download_test
  145. @unittest.skip('IE broken')
  146. class TestTedSubtitles(BaseTestSubtitles):
  147. url = 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html'
  148. IE = TedTalkIE
  149. def test_allsubtitles(self):
  150. self.DL.params['writesubtitles'] = True
  151. self.DL.params['allsubtitles'] = True
  152. subtitles = self.getSubtitles()
  153. self.assertTrue(len(subtitles.keys()) >= 28)
  154. self.assertEqual(md5(subtitles['en']), '4262c1665ff928a2dada178f62cb8d14')
  155. self.assertEqual(md5(subtitles['fr']), '66a63f7f42c97a50f8c0e90bc7797bb5')
  156. for lang in ['es', 'fr', 'de']:
  157. self.assertTrue(subtitles.get(lang) is not None, f'Subtitles for \'{lang}\' not extracted')
  158. @is_download_test
  159. class TestVimeoSubtitles(BaseTestSubtitles):
  160. url = 'http://vimeo.com/76979871'
  161. IE = VimeoIE
  162. def test_allsubtitles(self):
  163. self.DL.params['writesubtitles'] = True
  164. self.DL.params['allsubtitles'] = True
  165. subtitles = self.getSubtitles()
  166. self.assertEqual(set(subtitles.keys()), {'de', 'en', 'es', 'fr'})
  167. self.assertEqual(md5(subtitles['en']), '386cbc9320b94e25cb364b97935e5dd1')
  168. self.assertEqual(md5(subtitles['fr']), 'c9b69eef35bc6641c0d4da8a04f9dfac')
  169. def test_nosubtitles(self):
  170. self.DL.expect_warning('video doesn\'t have subtitles')
  171. self.url = 'http://vimeo.com/68093876'
  172. self.DL.params['writesubtitles'] = True
  173. self.DL.params['allsubtitles'] = True
  174. subtitles = self.getSubtitles()
  175. self.assertFalse(subtitles)
  176. @is_download_test
  177. @unittest.skip('IE broken')
  178. class TestWallaSubtitles(BaseTestSubtitles):
  179. url = 'http://vod.walla.co.il/movie/2705958/the-yes-men'
  180. IE = WallaIE
  181. def test_allsubtitles(self):
  182. self.DL.expect_warning('Automatic Captions not supported by this server')
  183. self.DL.params['writesubtitles'] = True
  184. self.DL.params['allsubtitles'] = True
  185. subtitles = self.getSubtitles()
  186. self.assertEqual(set(subtitles.keys()), {'heb'})
  187. self.assertEqual(md5(subtitles['heb']), 'e758c5d7cb982f6bef14f377ec7a3920')
  188. def test_nosubtitles(self):
  189. self.DL.expect_warning('video doesn\'t have subtitles')
  190. self.url = 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one'
  191. self.DL.params['writesubtitles'] = True
  192. self.DL.params['allsubtitles'] = True
  193. subtitles = self.getSubtitles()
  194. self.assertFalse(subtitles)
  195. @is_download_test
  196. @unittest.skip('IE broken')
  197. class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
  198. url = 'http://www.ceskatelevize.cz/ivysilani/10600540290-u6-uzasny-svet-techniky'
  199. IE = CeskaTelevizeIE
  200. def test_allsubtitles(self):
  201. self.DL.expect_warning('Automatic Captions not supported by this server')
  202. self.DL.params['writesubtitles'] = True
  203. self.DL.params['allsubtitles'] = True
  204. subtitles = self.getSubtitles()
  205. self.assertEqual(set(subtitles.keys()), {'cs'})
  206. self.assertTrue(len(subtitles['cs']) > 20000)
  207. def test_nosubtitles(self):
  208. self.DL.expect_warning('video doesn\'t have subtitles')
  209. self.url = 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220'
  210. self.DL.params['writesubtitles'] = True
  211. self.DL.params['allsubtitles'] = True
  212. subtitles = self.getSubtitles()
  213. self.assertFalse(subtitles)
  214. @is_download_test
  215. @unittest.skip('IE broken')
  216. class TestLyndaSubtitles(BaseTestSubtitles):
  217. url = 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html'
  218. IE = LyndaIE
  219. def test_allsubtitles(self):
  220. self.DL.params['writesubtitles'] = True
  221. self.DL.params['allsubtitles'] = True
  222. subtitles = self.getSubtitles()
  223. self.assertEqual(set(subtitles.keys()), {'en'})
  224. self.assertEqual(md5(subtitles['en']), '09bbe67222259bed60deaa26997d73a7')
  225. @is_download_test
  226. @unittest.skip('IE broken')
  227. class TestNPOSubtitles(BaseTestSubtitles):
  228. url = 'http://www.npo.nl/nos-journaal/28-08-2014/POW_00722860'
  229. IE = NPOIE
  230. def test_allsubtitles(self):
  231. self.DL.params['writesubtitles'] = True
  232. self.DL.params['allsubtitles'] = True
  233. subtitles = self.getSubtitles()
  234. self.assertEqual(set(subtitles.keys()), {'nl'})
  235. self.assertEqual(md5(subtitles['nl']), 'fc6435027572b63fb4ab143abd5ad3f4')
  236. @is_download_test
  237. @unittest.skip('IE broken')
  238. class TestMTVSubtitles(BaseTestSubtitles):
  239. url = 'http://www.cc.com/video-clips/p63lk0/adam-devine-s-house-party-chasing-white-swans'
  240. IE = ComedyCentralIE
  241. def getInfoDict(self):
  242. return super().getInfoDict()['entries'][0]
  243. def test_allsubtitles(self):
  244. self.DL.params['writesubtitles'] = True
  245. self.DL.params['allsubtitles'] = True
  246. subtitles = self.getSubtitles()
  247. self.assertEqual(set(subtitles.keys()), {'en'})
  248. self.assertEqual(md5(subtitles['en']), '78206b8d8a0cfa9da64dc026eea48961')
  249. @is_download_test
  250. class TestNRKSubtitles(BaseTestSubtitles):
  251. url = 'http://tv.nrk.no/serie/ikke-gjoer-dette-hjemme/DMPV73000411/sesong-2/episode-1'
  252. IE = NRKTVIE
  253. def test_allsubtitles(self):
  254. self.DL.params['writesubtitles'] = True
  255. self.DL.params['allsubtitles'] = True
  256. subtitles = self.getSubtitles()
  257. self.assertEqual(set(subtitles.keys()), {'nb-ttv'})
  258. self.assertEqual(md5(subtitles['nb-ttv']), '67e06ff02d0deaf975e68f6cb8f6a149')
  259. @is_download_test
  260. class TestRaiPlaySubtitles(BaseTestSubtitles):
  261. IE = RaiPlayIE
  262. def test_subtitles_key(self):
  263. self.url = 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html'
  264. self.DL.params['writesubtitles'] = True
  265. self.DL.params['allsubtitles'] = True
  266. subtitles = self.getSubtitles()
  267. self.assertEqual(set(subtitles.keys()), {'it'})
  268. self.assertEqual(md5(subtitles['it']), 'b1d90a98755126b61e667567a1f6680a')
  269. def test_subtitles_array_key(self):
  270. self.url = 'https://www.raiplay.it/video/2020/12/Report---04-01-2021-2e90f1de-8eee-4de4-ac0e-78d21db5b600.html'
  271. self.DL.params['writesubtitles'] = True
  272. self.DL.params['allsubtitles'] = True
  273. subtitles = self.getSubtitles()
  274. self.assertEqual(set(subtitles.keys()), {'it'})
  275. self.assertEqual(md5(subtitles['it']), '4b3264186fbb103508abe5311cfcb9cd')
  276. @is_download_test
  277. @unittest.skip('IE broken - DRM only')
  278. class TestVikiSubtitles(BaseTestSubtitles):
  279. url = 'http://www.viki.com/videos/1060846v-punch-episode-18'
  280. IE = VikiIE
  281. def test_allsubtitles(self):
  282. self.DL.params['writesubtitles'] = True
  283. self.DL.params['allsubtitles'] = True
  284. subtitles = self.getSubtitles()
  285. self.assertEqual(set(subtitles.keys()), {'en'})
  286. self.assertEqual(md5(subtitles['en']), '53cb083a5914b2d84ef1ab67b880d18a')
  287. @is_download_test
  288. class TestThePlatformSubtitles(BaseTestSubtitles):
  289. # from http://www.3playmedia.com/services-features/tools/integrations/theplatform/
  290. # (see http://theplatform.com/about/partners/type/subtitles-closed-captioning/)
  291. url = 'theplatform:JFUjUE1_ehvq'
  292. IE = ThePlatformIE
  293. def test_allsubtitles(self):
  294. self.DL.params['writesubtitles'] = True
  295. self.DL.params['allsubtitles'] = True
  296. subtitles = self.getSubtitles()
  297. self.assertEqual(set(subtitles.keys()), {'en'})
  298. self.assertEqual(md5(subtitles['en']), '97e7670cbae3c4d26ae8bcc7fdd78d4b')
  299. @is_download_test
  300. @unittest.skip('IE broken')
  301. class TestThePlatformFeedSubtitles(BaseTestSubtitles):
  302. url = 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207'
  303. IE = ThePlatformFeedIE
  304. def test_allsubtitles(self):
  305. self.DL.params['writesubtitles'] = True
  306. self.DL.params['allsubtitles'] = True
  307. subtitles = self.getSubtitles()
  308. self.assertEqual(set(subtitles.keys()), {'en'})
  309. self.assertEqual(md5(subtitles['en']), '48649a22e82b2da21c9a67a395eedade')
  310. @is_download_test
  311. class TestRtveSubtitles(BaseTestSubtitles):
  312. url = 'http://www.rtve.es/alacarta/videos/los-misterios-de-laura/misterios-laura-capitulo-32-misterio-del-numero-17-2-parte/2428621/'
  313. IE = RTVEALaCartaIE
  314. def test_allsubtitles(self):
  315. print('Skipping, only available from Spain')
  316. return
  317. self.DL.params['writesubtitles'] = True
  318. self.DL.params['allsubtitles'] = True
  319. subtitles = self.getSubtitles()
  320. self.assertEqual(set(subtitles.keys()), {'es'})
  321. self.assertEqual(md5(subtitles['es']), '69e70cae2d40574fb7316f31d6eb7fca')
  322. @is_download_test
  323. class TestDemocracynowSubtitles(BaseTestSubtitles):
  324. url = 'http://www.democracynow.org/shows/2015/7/3'
  325. IE = DemocracynowIE
  326. def test_allsubtitles(self):
  327. self.DL.params['writesubtitles'] = True
  328. self.DL.params['allsubtitles'] = True
  329. subtitles = self.getSubtitles()
  330. self.assertEqual(set(subtitles.keys()), {'en'})
  331. self.assertEqual(md5(subtitles['en']), 'a3cc4c0b5eadd74d9974f1c1f5101045')
  332. def test_subtitles_in_page(self):
  333. self.url = 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree'
  334. self.DL.params['writesubtitles'] = True
  335. self.DL.params['allsubtitles'] = True
  336. subtitles = self.getSubtitles()
  337. self.assertEqual(set(subtitles.keys()), {'en'})
  338. self.assertEqual(md5(subtitles['en']), 'a3cc4c0b5eadd74d9974f1c1f5101045')
  339. @is_download_test
  340. class TestPBSSubtitles(BaseTestSubtitles):
  341. url = 'https://www.pbs.org/video/how-fantasy-reflects-our-world-picecq/'
  342. IE = PBSIE
  343. def test_allsubtitles(self):
  344. self.DL.params['writesubtitles'] = True
  345. self.DL.params['allsubtitles'] = True
  346. subtitles = self.getSubtitles()
  347. self.assertEqual(set(subtitles.keys()), {'en'})
  348. def test_subtitles_dfxp_format(self):
  349. self.DL.params['writesubtitles'] = True
  350. self.DL.params['subtitlesformat'] = 'dfxp'
  351. subtitles = self.getSubtitles()
  352. self.assertIn(md5(subtitles['en']), ['643b034254cdc3768ff1e750b6b5873b'])
  353. def test_subtitles_vtt_format(self):
  354. self.DL.params['writesubtitles'] = True
  355. self.DL.params['subtitlesformat'] = 'vtt'
  356. subtitles = self.getSubtitles()
  357. self.assertIn(
  358. md5(subtitles['en']), ['937a05711555b165d4c55a9667017045', 'f49ea998d6824d94959c8152a368ff73'])
  359. def test_subtitles_srt_format(self):
  360. self.DL.params['writesubtitles'] = True
  361. self.DL.params['subtitlesformat'] = 'srt'
  362. subtitles = self.getSubtitles()
  363. self.assertIn(md5(subtitles['en']), ['2082c21b43759d9bf172931b2f2ca371'])
  364. def test_subtitles_sami_format(self):
  365. self.DL.params['writesubtitles'] = True
  366. self.DL.params['subtitlesformat'] = 'sami'
  367. subtitles = self.getSubtitles()
  368. self.assertIn(md5(subtitles['en']), ['4256b16ac7da6a6780fafd04294e85cd'])
  369. if __name__ == '__main__':
  370. unittest.main()