123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312 |
- import base64
- import binascii
- import json
- import time
- import uuid
- from .common import InfoExtractor
- from ..dependencies import Cryptodome
- from ..utils import (
- ExtractorError,
- int_or_none,
- jwt_decode_hs256,
- traverse_obj,
- try_call,
- url_basename,
- url_or_none,
- urlencode_postdata,
- variadic,
- )
- class WrestleUniverseBaseIE(InfoExtractor):
- _NETRC_MACHINE = 'wrestleuniverse'
- _VALID_URL_TMPL = r'https?://(?:www\.)?wrestle-universe\.com/(?:(?P<lang>\w{2})/)?%s/(?P<id>\w+)'
- _API_HOST = 'api.wrestle-universe.com'
- _API_PATH = None
- _REAL_TOKEN = None
- _TOKEN_EXPIRY = None
- _REFRESH_TOKEN = None
- _DEVICE_ID = None
- _LOGIN_QUERY = {'key': 'AIzaSyCaRPBsDQYVDUWWBXjsTrHESi2r_F3RAdA'}
- _LOGIN_HEADERS = {
- 'Accept': '*/*',
- 'Content-Type': 'application/json',
- 'X-Client-Version': 'Chrome/JsCore/9.9.4/FirebaseCore-web',
- 'X-Firebase-gmpid': '1:307308870738:web:820f38fe5150c8976e338b',
- 'Referer': 'https://www.wrestle-universe.com/',
- 'Origin': 'https://www.wrestle-universe.com',
- }
- @property
- def _TOKEN(self):
- if not self._REAL_TOKEN or not self._TOKEN_EXPIRY:
- token = try_call(lambda: self._get_cookies('https://www.wrestle-universe.com/')['token'].value)
- if not token and not self._REFRESH_TOKEN:
- self.raise_login_required()
- self._TOKEN = token
- if not self._REAL_TOKEN or self._TOKEN_EXPIRY <= int(time.time()):
- if not self._REFRESH_TOKEN:
- raise ExtractorError(
- 'Expired token. Refresh your cookies in browser and try again', expected=True)
- self._refresh_token()
- return self._REAL_TOKEN
- @_TOKEN.setter
- def _TOKEN(self, value):
- self._REAL_TOKEN = value
- expiry = traverse_obj(value, ({jwt_decode_hs256}, 'exp', {int_or_none}))
- if not expiry:
- raise ExtractorError('There was a problem with the auth token')
- self._TOKEN_EXPIRY = expiry
- def _perform_login(self, username, password):
- login = self._download_json(
- 'https://identitytoolkit.googleapis.com/v1/accounts:signInWithPassword', None,
- 'Logging in', query=self._LOGIN_QUERY, headers=self._LOGIN_HEADERS, data=json.dumps({
- 'returnSecureToken': True,
- 'email': username,
- 'password': password,
- }, separators=(',', ':')).encode(), expected_status=400)
- token = traverse_obj(login, ('idToken', {str}))
- if not token:
- raise ExtractorError(
- f'Unable to log in: {traverse_obj(login, ("error", "message"))}', expected=True)
- self._REFRESH_TOKEN = traverse_obj(login, ('refreshToken', {str}))
- if not self._REFRESH_TOKEN:
- self.report_warning('No refresh token was granted')
- self._TOKEN = token
- def _real_initialize(self):
- if self._DEVICE_ID:
- return
- self._DEVICE_ID = self._configuration_arg('device_id', [None], ie_key=self._NETRC_MACHINE)[0]
- if not self._DEVICE_ID:
- self._DEVICE_ID = self.cache.load(self._NETRC_MACHINE, 'device_id')
- if self._DEVICE_ID:
- return
- self._DEVICE_ID = str(uuid.uuid4())
- self.cache.store(self._NETRC_MACHINE, 'device_id', self._DEVICE_ID)
- def _refresh_token(self):
- refresh = self._download_json(
- 'https://securetoken.googleapis.com/v1/token', None, 'Refreshing token',
- query=self._LOGIN_QUERY, data=urlencode_postdata({
- 'grant_type': 'refresh_token',
- 'refresh_token': self._REFRESH_TOKEN,
- }), headers={
- **self._LOGIN_HEADERS,
- 'Content-Type': 'application/x-www-form-urlencoded',
- })
- if traverse_obj(refresh, ('refresh_token', {str})):
- self._REFRESH_TOKEN = refresh['refresh_token']
- token = traverse_obj(refresh, 'access_token', 'id_token', expected_type=str)
- if not token:
- raise ExtractorError('No auth token returned from refresh request')
- self._TOKEN = token
- def _call_api(self, video_id, param='', msg='API', auth=True, data=None, query={}, fatal=True):
- headers = {'CA-CID': ''}
- if data:
- headers['Content-Type'] = 'application/json;charset=utf-8'
- data = json.dumps(data, separators=(',', ':')).encode()
- if auth and self._TOKEN:
- headers['Authorization'] = f'Bearer {self._TOKEN}'
- return self._download_json(
- f'https://{self._API_HOST}/v1/{self._API_PATH}/{video_id}{param}', video_id,
- note=f'Downloading {msg} JSON', errnote=f'Failed to download {msg} JSON',
- data=data, headers=headers, query=query, fatal=fatal)
- def _call_encrypted_api(self, video_id, param='', msg='API', data={}, query={}, fatal=True):
- if not Cryptodome.RSA:
- raise ExtractorError('pycryptodomex not found. Please install', expected=True)
- private_key = Cryptodome.RSA.generate(2048)
- cipher = Cryptodome.PKCS1_OAEP.new(private_key, hashAlgo=Cryptodome.SHA1)
- def decrypt(data):
- if not data:
- return None
- try:
- return cipher.decrypt(base64.b64decode(data)).decode()
- except (ValueError, binascii.Error) as e:
- raise ExtractorError(f'Could not decrypt data: {e}')
- token = base64.b64encode(private_key.public_key().export_key('DER')).decode()
- api_json = self._call_api(video_id, param, msg, data={
- 'deviceId': self._DEVICE_ID,
- 'token': token,
- **data,
- }, query=query, fatal=fatal)
- return api_json, decrypt
- def _download_metadata(self, url, video_id, lang, props_keys):
- metadata = self._call_api(video_id, msg='metadata', query={'al': lang or 'ja'}, auth=False, fatal=False)
- if not metadata:
- webpage = self._download_webpage(url, video_id)
- nextjs_data = self._search_nextjs_data(webpage, video_id, fatal=False)
- metadata = traverse_obj(nextjs_data, (
- 'props', 'pageProps', *variadic(props_keys, (str, bytes, dict, set)), {dict})) or {}
- return metadata
- def _get_formats(self, data, path, video_id=None):
- hls_url = traverse_obj(data, path, get_all=False)
- if not hls_url and not data.get('canWatch'):
- self.raise_no_formats(
- 'This account does not have access to the requested content', expected=True)
- elif not hls_url:
- self.raise_no_formats('No supported formats found')
- return self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls', live=True)
- class WrestleUniverseVODIE(WrestleUniverseBaseIE):
- _VALID_URL = WrestleUniverseBaseIE._VALID_URL_TMPL % 'videos'
- _TESTS = [{
- 'url': 'https://www.wrestle-universe.com/en/videos/dp8mpjmcKfxzUhEHM2uFws',
- 'info_dict': {
- 'id': 'dp8mpjmcKfxzUhEHM2uFws',
- 'ext': 'mp4',
- 'title': 'The 3rd “Futari wa Princess” Max Heart Tournament',
- 'description': 'md5:318d5061e944797fbbb81d5c7dd00bf5',
- 'location': '埼玉・春日部ふれあいキューブ',
- 'channel': 'tjpw',
- 'duration': 7119,
- 'timestamp': 1674979200,
- 'upload_date': '20230129',
- 'thumbnail': 'https://image.asset.wrestle-universe.com/8FjD67P8rZc446RBQs5RBN/8FjD67P8rZc446RBQs5RBN',
- 'chapters': 'count:7',
- 'cast': 'count:21',
- },
- 'params': {
- 'skip_download': 'm3u8',
- },
- }]
- _API_PATH = 'videoEpisodes'
- def _real_extract(self, url):
- lang, video_id = self._match_valid_url(url).group('lang', 'id')
- metadata = self._download_metadata(url, video_id, lang, 'videoEpisodeFallbackData')
- video_data = self._call_api(video_id, ':watch', 'watch', data={'deviceId': self._DEVICE_ID})
- return {
- 'id': video_id,
- 'formats': self._get_formats(video_data, ('protocolHls', 'url', {url_or_none}), video_id),
- **traverse_obj(metadata, {
- 'title': ('displayName', {str}),
- 'description': ('description', {str}),
- 'channel': ('labels', 'group', {str}),
- 'location': ('labels', 'venue', {str}),
- 'timestamp': ('watchStartTime', {int_or_none}),
- 'thumbnail': ('keyVisualUrl', {url_or_none}),
- 'cast': ('casts', ..., 'displayName', {str}),
- 'duration': ('duration', {int}),
- 'chapters': ('videoChapters', lambda _, v: isinstance(v.get('start'), int), {
- 'title': ('displayName', {str}),
- 'start_time': ('start', {int}),
- 'end_time': ('end', {int}),
- }),
- }),
- }
- class WrestleUniversePPVIE(WrestleUniverseBaseIE):
- _VALID_URL = WrestleUniverseBaseIE._VALID_URL_TMPL % 'lives'
- _TESTS = [{
- 'note': 'HLS AES-128 key obtained via API',
- 'url': 'https://www.wrestle-universe.com/en/lives/buH9ibbfhdJAY4GKZcEuJX',
- 'info_dict': {
- 'id': 'buH9ibbfhdJAY4GKZcEuJX',
- 'ext': 'mp4',
- 'title': '【PPV】Beyond the origins, into the future',
- 'description': 'md5:9a872db68cd09be4a1e35a3ee8b0bdfc',
- 'channel': 'tjpw',
- 'location': '東京・Twin Box AKIHABARA',
- 'duration': 10098,
- 'timestamp': 1675076400,
- 'upload_date': '20230130',
- 'thumbnail': 'https://image.asset.wrestle-universe.com/rJs2m7cBaLXrwCcxMdQGRM/rJs2m7cBaLXrwCcxMdQGRM',
- 'thumbnails': 'count:3',
- 'hls_aes': {
- 'key': '5633184acd6e43f1f1ac71c6447a4186',
- 'iv': '5bac71beb33197d5600337ce86de7862',
- },
- },
- 'params': {
- 'skip_download': 'm3u8',
- },
- 'skip': 'No longer available',
- }, {
- 'note': 'unencrypted HLS',
- 'url': 'https://www.wrestle-universe.com/en/lives/wUG8hP5iApC63jbtQzhVVx',
- 'info_dict': {
- 'id': 'wUG8hP5iApC63jbtQzhVVx',
- 'ext': 'mp4',
- 'title': 'GRAND PRINCESS \'22',
- 'description': 'md5:e4f43d0d4262de3952ff34831bc99858',
- 'channel': 'tjpw',
- 'location': '東京・両国国技館',
- 'duration': 18044,
- 'timestamp': 1647665400,
- 'upload_date': '20220319',
- 'thumbnail': 'https://image.asset.wrestle-universe.com/i8jxSTCHPfdAKD4zN41Psx/i8jxSTCHPfdAKD4zN41Psx',
- 'thumbnails': 'count:3',
- },
- 'params': {
- 'skip_download': 'm3u8',
- },
- }, {
- 'note': 'manifest provides live-a (partial) and live-b (full) streams',
- 'url': 'https://www.wrestle-universe.com/en/lives/umc99R9XsexXrxr9VjTo9g',
- 'only_matching': True,
- }]
- _API_PATH = 'events'
- def _real_extract(self, url):
- lang, video_id = self._match_valid_url(url).group('lang', 'id')
- metadata = self._download_metadata(url, video_id, lang, 'eventFallbackData')
- info = {
- 'id': video_id,
- **traverse_obj(metadata, {
- 'title': ('displayName', {str}),
- 'description': ('description', {str}),
- 'channel': ('labels', 'group', {str}),
- 'location': ('labels', 'venue', {str}),
- 'timestamp': ('startTime', {int_or_none}),
- 'thumbnails': (('keyVisualUrl', 'alterKeyVisualUrl', 'heroKeyVisualUrl'), {'url': {url_or_none}}),
- }),
- }
- ended_time = traverse_obj(metadata, ('endedTime', {int_or_none}))
- if info.get('timestamp') and ended_time:
- info['duration'] = ended_time - info['timestamp']
- video_data, decrypt = self._call_encrypted_api(
- video_id, ':watchArchive', 'watch archive', data={'method': 1})
- # 'chromecastUrls' can be only partial videos, avoid
- info['formats'] = self._get_formats(video_data, ('hls', (('urls', ...), 'url'), {url_or_none}), video_id)
- for f in info['formats']:
- # bitrates are exaggerated in PPV playlists, so avoid wrong/huge filesize_approx values
- if f.get('tbr'):
- f['tbr'] = int(f['tbr'] / 2.5)
- # prefer variants with the same basename as the master playlist to avoid partial streams
- f['format_id'] = url_basename(f['url']).partition('.')[0]
- if not f['format_id'].startswith(url_basename(f['manifest_url']).partition('.')[0]):
- f['preference'] = -10
- hls_aes_key = traverse_obj(video_data, ('hls', 'key', {decrypt}))
- if hls_aes_key:
- info['hls_aes'] = {
- 'key': hls_aes_key,
- 'iv': traverse_obj(video_data, ('hls', 'iv', {decrypt})),
- }
- elif traverse_obj(video_data, ('hls', 'encryptType', {int})):
- self.report_warning('HLS AES-128 key was not found in API response')
- return info
|