123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390 |
- import itertools
- from .common import InfoExtractor
- from ..utils import (
- determine_ext,
- extract_attributes,
- int_or_none,
- lowercase_escape,
- parse_qs,
- traverse_obj,
- try_get,
- url_or_none,
- )
- class YandexVideoIE(InfoExtractor):
- _VALID_URL = r'''(?x)
- https?://
- (?:
- yandex\.ru(?:/(?:portal/(?:video|efir)|efir))?/?\?.*?stream_id=|
- frontend\.vh\.yandex\.ru/player/
- )
- (?P<id>(?:[\da-f]{32}|[\w-]{12}))
- '''
- _TESTS = [{
- 'url': 'https://yandex.ru/portal/video?stream_id=4dbb36ec4e0526d58f9f2dc8f0ecf374',
- 'info_dict': {
- 'id': '4dbb36ec4e0526d58f9f2dc8f0ecf374',
- 'ext': 'mp4',
- 'title': 'Русский Вудсток - главный рок-фест в истории СССР / вДудь',
- 'description': 'md5:7d6b8d4bc4a3b9a56499916c1ea5b5fa',
- 'thumbnail': r're:^https?://',
- 'timestamp': 1549972939,
- 'duration': 5575,
- 'age_limit': 18,
- 'upload_date': '20190212',
- 'view_count': int,
- 'like_count': int,
- 'dislike_count': int,
- },
- 'params': {'skip_download': 'm3u8'},
- }, {
- 'url': 'https://yandex.ru/portal/efir?stream_id=4dbb262b4fe5cf15a215de4f34eee34d&from=morda',
- 'only_matching': True,
- }, {
- 'url': 'https://yandex.ru/?stream_id=4dbb262b4fe5cf15a215de4f34eee34d',
- 'only_matching': True,
- }, {
- 'url': 'https://frontend.vh.yandex.ru/player/4dbb262b4fe5cf15a215de4f34eee34d?from=morda',
- 'only_matching': True,
- }, {
- # vod-episode, series episode
- 'url': 'https://yandex.ru/portal/video?stream_id=45b11db6e4b68797919c93751a938cee',
- 'only_matching': True,
- }, {
- # episode, sports
- 'url': 'https://yandex.ru/?stream_channel=1538487871&stream_id=4132a07f71fb0396be93d74b3477131d',
- 'only_matching': True,
- }, {
- # DASH with DRM
- 'url': 'https://yandex.ru/portal/video?from=morda&stream_id=485a92d94518d73a9d0ff778e13505f8',
- 'only_matching': True,
- }, {
- 'url': 'https://yandex.ru/efir?stream_active=watching&stream_id=v7a2dZ-v5mSI&from_block=efir_newtab',
- 'only_matching': True,
- }]
- def _real_extract(self, url):
- video_id = self._match_id(url)
- player = try_get((self._download_json(
- 'https://frontend.vh.yandex.ru/graphql', video_id, data=('''{
- player(content_id: "%s") {
- computed_title
- content_url
- description
- dislikes
- duration
- likes
- program_title
- release_date
- release_date_ut
- release_year
- restriction_age
- season
- start_time
- streams
- thumbnail
- title
- views_count
- }
- }''' % video_id).encode(), fatal=False)), lambda x: x['player']['content']) # noqa: UP031
- if not player or player.get('error'):
- player = self._download_json(
- f'https://frontend.vh.yandex.ru/v23/player/{video_id}.json',
- video_id, query={
- 'stream_options': 'hires',
- 'disable_trackings': 1,
- })
- content = player['content']
- title = content.get('title') or content['computed_title']
- formats = []
- streams = content.get('streams') or []
- streams.append({'url': content.get('content_url')})
- for stream in streams:
- content_url = url_or_none(stream.get('url'))
- if not content_url:
- continue
- ext = determine_ext(content_url)
- if ext == 'ismc':
- continue
- elif ext == 'm3u8':
- formats.extend(self._extract_m3u8_formats(
- content_url, video_id, 'mp4',
- 'm3u8_native', m3u8_id='hls', fatal=False))
- elif ext == 'mpd':
- formats.extend(self._extract_mpd_formats(
- content_url, video_id, mpd_id='dash', fatal=False))
- else:
- formats.append({'url': content_url})
- timestamp = (int_or_none(content.get('release_date'))
- or int_or_none(content.get('release_date_ut'))
- or int_or_none(content.get('start_time')))
- season = content.get('season') or {}
- return {
- 'id': video_id,
- 'title': title,
- 'description': content.get('description'),
- 'thumbnail': content.get('thumbnail'),
- 'timestamp': timestamp,
- 'duration': int_or_none(content.get('duration')),
- 'series': content.get('program_title'),
- 'age_limit': int_or_none(content.get('restriction_age')),
- 'view_count': int_or_none(content.get('views_count')),
- 'like_count': int_or_none(content.get('likes')),
- 'dislike_count': int_or_none(content.get('dislikes')),
- 'season_number': int_or_none(season.get('season_number')),
- 'season_id': season.get('id'),
- 'release_year': int_or_none(content.get('release_year')),
- 'formats': formats,
- }
- class YandexVideoPreviewIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?yandex\.\w{2,3}(?:\.(?:am|ge|il|tr))?/video/preview(?:/?\?.*?filmId=|/)(?P<id>\d+)'
- _TESTS = [{ # Odnoklassniki
- 'url': 'https://yandex.ru/video/preview/?filmId=10682852472978372885&text=summer',
- 'info_dict': {
- 'id': '1352565459459',
- 'ext': 'mp4',
- 'like_count': int,
- 'upload_date': '20191202',
- 'age_limit': 0,
- 'duration': 196,
- 'thumbnail': 'https://i.mycdn.me/videoPreview?id=544866765315&type=37&idx=13&tkn=TY5qjLYZHxpmcnK8U2LgzYkgmaU&fn=external_8',
- 'uploader_id': '481054701571',
- 'title': 'LOFT - summer, summer, summer HD',
- 'uploader': 'АРТЁМ КУДРОВ',
- },
- }, { # youtube
- 'url': 'https://yandex.ru/video/preview/?filmId=4479424425337895262&source=main_redirect&text=видео&utm_source=main_stripe_big',
- 'only_matching': True,
- }, { # YandexVideo
- 'url': 'https://yandex.ru/video/preview/5275069442094787341',
- 'only_matching': True,
- }, { # youtube
- 'url': 'https://yandex.ru/video/preview/?filmId=16658118429797832897&from=tabbar&p=1&text=%D0%BF%D1%80%D0%BE%D1%81%D0%BC%D0%BE%D1%82%D1%80+%D1%84%D1%80%D0%B0%D0%B3%D0%BC%D0%B5%D0%BD%D1%82%D0%B0+%D0%BC%D0%B0%D0%BB%D0%B5%D0%BD%D1%8C%D0%BA%D0%B8%D0%B9+%D0%BF%D1%80%D0%B8%D0%BD%D1%86+%D0%BC%D1%8B+%D0%B2+%D0%BE%D1%82%D0%B2%D0%B5%D1%82%D0%B5+%D0%B7%D0%B0+%D1%82%D0%B5%D1%85+%D0%BA%D0%BE%D0%B3%D0%BE+%D0%BF%D1%80%D0%B8%D1%80%D1%83%D1%87%D0%B8%D0%BB%D0%B8',
- 'only_matching': True,
- }, { # Odnoklassniki
- 'url': 'https://yandex.ru/video/preview/?text=Francis%20Lai%20-%20Le%20Bon%20Et%20Les%20MC)chants&path=wizard&parent-reqid=1643208087979310-1481782809207673478-sas3-0931-2f9-sas-l7-balancer-8080-BAL-9380&wiz_type=vital&filmId=12508152936505397283',
- 'only_matching': True,
- }, { # Odnoklassniki
- 'url': 'https://yandex.com/video/preview/?text=dossier%2051%20film%201978&path=yandex_search&parent-reqid=1664361087754492-8727541069609384458-sas2-0340-sas-l7-balancer-8080-BAL-8045&noreask=1&from_type=vast&filmId=5794987234584444632',
- 'only_matching': True,
- }]
- def _real_extract(self, url):
- video_id = self._match_id(url)
- webpage = self._download_webpage(url, video_id)
- data_raw = self._search_regex(r'window.Ya.__inline_params__\s*=\s*JSON.parse\(\'([^"]+?\\u0022video\\u0022:[^"]+?})\'\);', webpage, 'data_raw')
- data_json = self._parse_json(data_raw, video_id, transform_source=lowercase_escape)
- return self.url_result(data_json['video']['url'])
- class ZenYandexIE(InfoExtractor):
- _VALID_URL = r'https?://(zen\.yandex|dzen)\.ru(?:/video)?/(media|watch)/(?:(?:id/[^/]+/|[^/]+/)(?:[a-z0-9-]+)-)?(?P<id>[a-z0-9-]+)'
- _TESTS = [{
- 'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7',
- 'info_dict': {
- 'id': '60c7c443da18892ebfe85ed7',
- 'ext': 'mp4',
- 'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
- 'description': 'md5:8684912f6086f298f8078d4af0e8a600',
- 'thumbnail': 're:^https://avatars.dzeninfra.ru/',
- 'uploader': 'AcademeG DailyStream',
- },
- 'params': {
- 'skip_download': 'm3u8',
- 'format': 'bestvideo',
- },
- 'skip': 'The page does not exist',
- }, {
- 'url': 'https://dzen.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7',
- 'info_dict': {
- 'id': '60c7c443da18892ebfe85ed7',
- 'ext': 'mp4',
- 'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
- 'description': 'md5:8684912f6086f298f8078d4af0e8a600',
- 'thumbnail': r're:^https://avatars\.dzeninfra\.ru/',
- 'uploader': 'AcademeG DailyStream',
- 'upload_date': '20191111',
- 'timestamp': 1573465585,
- },
- 'params': {'skip_download': 'm3u8'},
- }, {
- 'url': 'https://zen.yandex.ru/video/watch/6002240ff8b1af50bb2da5e3',
- 'info_dict': {
- 'id': '6002240ff8b1af50bb2da5e3',
- 'ext': 'mp4',
- 'title': 'Извержение вулкана из спичек: зрелищный опыт',
- 'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
- 'thumbnail': r're:^https://avatars\.dzeninfra\.ru/',
- 'uploader': 'TechInsider',
- 'timestamp': 1611378221,
- 'upload_date': '20210123',
- },
- 'params': {'skip_download': 'm3u8'},
- }, {
- 'url': 'https://dzen.ru/video/watch/6002240ff8b1af50bb2da5e3',
- 'info_dict': {
- 'id': '6002240ff8b1af50bb2da5e3',
- 'ext': 'mp4',
- 'title': 'Извержение вулкана из спичек: зрелищный опыт',
- 'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
- 'thumbnail': 're:^https://avatars.dzeninfra.ru/',
- 'uploader': 'TechInsider',
- 'upload_date': '20210123',
- 'timestamp': 1611378221,
- },
- 'params': {'skip_download': 'm3u8'},
- }, {
- 'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360',
- 'only_matching': True,
- }, {
- 'url': 'https://dzen.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360',
- 'only_matching': True,
- }]
- def _real_extract(self, url):
- video_id = self._match_id(url)
- webpage = self._download_webpage(url, video_id)
- redirect = self._search_json(r'var it\s*=', webpage, 'redirect', id, default={}).get('retpath')
- if redirect:
- video_id = self._match_id(redirect)
- webpage = self._download_webpage(redirect, video_id, note='Redirecting')
- data_json = self._search_json(
- r'("data"\s*:|data\s*=)', webpage, 'metadata', video_id, contains_pattern=r'{["\']_*serverState_*video.+}')
- serverstate = self._search_regex(r'(_+serverState_+video-site_[^_]+_+)', webpage, 'server state')
- uploader = self._search_regex(r'(<a\s*class=["\']card-channel-link[^"\']+["\'][^>]+>)',
- webpage, 'uploader', default='<a>')
- uploader_name = extract_attributes(uploader).get('aria-label')
- item_id = traverse_obj(data_json, (serverstate, 'videoViewer', 'openedItemId', {str}))
- video_json = traverse_obj(data_json, (serverstate, 'videoViewer', 'items', item_id, {dict})) or {}
- formats, subtitles = [], {}
- for s_url in traverse_obj(video_json, ('video', 'streams', ..., {url_or_none})):
- ext = determine_ext(s_url)
- if ext == 'mpd':
- fmts, subs = self._extract_mpd_formats_and_subtitles(s_url, video_id, mpd_id='dash')
- elif ext == 'm3u8':
- fmts, subs = self._extract_m3u8_formats_and_subtitles(s_url, video_id, 'mp4')
- formats.extend(fmts)
- subtitles = self._merge_subtitles(subtitles, subs)
- return {
- 'id': video_id,
- 'title': video_json.get('title') or self._og_search_title(webpage),
- 'formats': formats,
- 'subtitles': subtitles,
- 'duration': int_or_none(video_json.get('duration')),
- 'view_count': int_or_none(video_json.get('views')),
- 'timestamp': int_or_none(video_json.get('publicationDate')),
- 'uploader': uploader_name or data_json.get('authorName') or try_get(data_json, lambda x: x['publisher']['name']),
- 'description': video_json.get('description') or self._og_search_description(webpage),
- 'thumbnail': self._og_search_thumbnail(webpage) or try_get(data_json, lambda x: x['og']['imageUrl']),
- }
- class ZenYandexChannelIE(InfoExtractor):
- _VALID_URL = r'https?://(zen\.yandex|dzen)\.ru/(?!media|video)(?:id/)?(?P<id>[a-z0-9-_]+)'
- _TESTS = [{
- 'url': 'https://zen.yandex.ru/tok_media',
- 'info_dict': {
- 'id': 'tok_media',
- 'title': 'СПЕКТР',
- 'description': 'md5:a9e5b3c247b7fe29fd21371a428bcf56',
- },
- 'playlist_mincount': 169,
- 'skip': 'The page does not exist',
- }, {
- 'url': 'https://dzen.ru/tok_media',
- 'info_dict': {
- 'id': 'tok_media',
- 'title': 'СПЕКТР',
- 'description': 'md5:a9e5b3c247b7fe29fd21371a428bcf56',
- },
- 'playlist_mincount': 169,
- 'skip': 'The page does not exist',
- }, {
- 'url': 'https://zen.yandex.ru/id/606fd806cc13cb3c58c05cf5',
- 'info_dict': {
- 'id': '606fd806cc13cb3c58c05cf5',
- 'description': 'md5:517b7c97d8ca92e940f5af65448fd928',
- 'title': 'AcademeG DailyStream',
- },
- 'playlist_mincount': 657,
- }, {
- # Test that the playlist extractor finishes extracting when the
- # channel has less than one page
- 'url': 'https://zen.yandex.ru/jony_me',
- 'info_dict': {
- 'id': 'jony_me',
- 'description': 'md5:ce0a5cad2752ab58701b5497835b2cc5',
- 'title': 'JONY ',
- },
- 'playlist_count': 18,
- }, {
- # Test that the playlist extractor finishes extracting when the
- # channel has more than one page of entries
- 'url': 'https://zen.yandex.ru/tatyanareva',
- 'info_dict': {
- 'id': 'tatyanareva',
- 'description': 'md5:40a1e51f174369ec3ba9d657734ac31f',
- 'title': 'Татьяна Рева',
- 'entries': 'maxcount:200',
- },
- 'playlist_mincount': 46,
- }, {
- 'url': 'https://dzen.ru/id/606fd806cc13cb3c58c05cf5',
- 'info_dict': {
- 'id': '606fd806cc13cb3c58c05cf5',
- 'title': 'AcademeG DailyStream',
- 'description': 'md5:517b7c97d8ca92e940f5af65448fd928',
- },
- 'playlist_mincount': 657,
- }]
- def _entries(self, item_id, server_state_json, server_settings_json):
- items = (traverse_obj(server_state_json, ('feed', 'items', ...))
- or traverse_obj(server_settings_json, ('exportData', 'items', ...)))
- more = (traverse_obj(server_state_json, ('links', 'more'))
- or traverse_obj(server_settings_json, ('exportData', 'more', 'link')))
- next_page_id = None
- for page in itertools.count(1):
- for item in items or []:
- if item.get('type') != 'gif':
- continue
- video_id = traverse_obj(item, 'publication_id', 'publicationId') or ''
- yield self.url_result(item['link'], ZenYandexIE, video_id.split(':')[-1])
- current_page_id = next_page_id
- next_page_id = traverse_obj(parse_qs(more), ('next_page_id', -1))
- if not all((more, items, next_page_id, next_page_id != current_page_id)):
- break
- data = self._download_json(more, item_id, note=f'Downloading Page {page}')
- items, more = data.get('items'), traverse_obj(data, ('more', 'link'))
- def _real_extract(self, url):
- item_id = self._match_id(url)
- webpage = self._download_webpage(url, item_id)
- redirect = self._search_json(
- r'var it\s*=', webpage, 'redirect', item_id, default={}).get('retpath')
- if redirect:
- item_id = self._match_id(redirect)
- webpage = self._download_webpage(redirect, item_id, note='Redirecting')
- data = self._search_json(
- r'("data"\s*:|data\s*=)', webpage, 'channel data', item_id, contains_pattern=r'{\"__serverState__.+}')
- server_state_json = traverse_obj(data, lambda k, _: k.startswith('__serverState__'), get_all=False)
- server_settings_json = traverse_obj(data, lambda k, _: k.startswith('__serverSettings__'), get_all=False)
- return self.playlist_result(
- self._entries(item_id, server_state_json, server_settings_json),
- item_id, traverse_obj(server_state_json, ('channel', 'source', 'title')),
- traverse_obj(server_state_json, ('channel', 'source', 'description')))
|