123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376 |
- import base64
- import itertools
- import urllib.parse
- from .common import InfoExtractor
- from ..compat import compat_ord
- from ..utils import (
- ExtractorError,
- int_or_none,
- parse_iso8601,
- strip_or_none,
- try_get,
- )
- class MixcloudBaseIE(InfoExtractor):
- def _call_api(self, object_type, object_fields, display_id, username, slug=None):
- lookup_key = object_type + 'Lookup'
- return self._download_json(
- 'https://app.mixcloud.com/graphql', display_id, query={
- 'query': '''{
- %s(lookup: {username: "%s"%s}) {
- %s
- }
- }''' % (lookup_key, username, f', slug: "{slug}"' if slug else '', object_fields), # noqa: UP031
- })['data'][lookup_key]
- class MixcloudIE(MixcloudBaseIE):
- _VALID_URL = r'https?://(?:(?:www|beta|m)\.)?mixcloud\.com/([^/]+)/(?!stream|uploads|favorites|listens|playlists)([^/]+)'
- IE_NAME = 'mixcloud'
- _TESTS = [{
- 'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/',
- 'info_dict': {
- 'id': 'dholbach_cryptkeeper',
- 'ext': 'm4a',
- 'title': 'Cryptkeeper',
- 'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
- 'uploader': 'Daniel Holbach',
- 'uploader_id': 'dholbach',
- 'thumbnail': r're:https?://.*\.jpg',
- 'view_count': int,
- 'timestamp': 1321359578,
- 'upload_date': '20111115',
- 'uploader_url': 'https://www.mixcloud.com/dholbach/',
- 'artist': 'Submorphics & Chino , Telekinesis, Porter Robinson, Enei, Breakage ft Jess Mills',
- 'duration': 3723,
- 'tags': [],
- 'comment_count': int,
- 'repost_count': int,
- 'like_count': int,
- },
- 'params': {'skip_download': 'm3u8'},
- }, {
- 'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
- 'info_dict': {
- 'id': 'gillespeterson_caribou-7-inch-vinyl-mix-chat',
- 'ext': 'mp3',
- 'title': 'Caribou 7 inch Vinyl Mix & Chat',
- 'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
- 'uploader': 'Gilles Peterson Worldwide',
- 'uploader_id': 'gillespeterson',
- 'thumbnail': 're:https?://.*',
- 'view_count': int,
- 'timestamp': 1422987057,
- 'upload_date': '20150203',
- 'uploader_url': 'https://www.mixcloud.com/gillespeterson/',
- 'duration': 2992,
- 'tags': [],
- 'comment_count': int,
- 'repost_count': int,
- 'like_count': int,
- },
- 'params': {'skip_download': '404 playback error on site'},
- }, {
- 'url': 'https://beta.mixcloud.com/RedLightRadio/nosedrip-15-red-light-radio-01-18-2016/',
- 'only_matching': True,
- }]
- _DECRYPTION_KEY = 'IFYOUWANTTHEARTISTSTOGETPAIDDONOTDOWNLOADFROMMIXCLOUD'
- @staticmethod
- def _decrypt_xor_cipher(key, ciphertext):
- """Encrypt/Decrypt XOR cipher. Both ways are possible because it's XOR."""
- return ''.join([
- chr(compat_ord(ch) ^ compat_ord(k))
- for ch, k in zip(ciphertext, itertools.cycle(key))])
- def _real_extract(self, url):
- username, slug = self._match_valid_url(url).groups()
- username, slug = urllib.parse.unquote(username), urllib.parse.unquote(slug)
- track_id = f'{username}_{slug}'
- cloudcast = self._call_api('cloudcast', '''audioLength
- comments(first: 100) {
- edges {
- node {
- comment
- created
- user {
- displayName
- username
- }
- }
- }
- totalCount
- }
- description
- favorites {
- totalCount
- }
- featuringArtistList
- isExclusive
- name
- owner {
- displayName
- url
- username
- }
- picture(width: 1024, height: 1024) {
- url
- }
- plays
- publishDate
- reposts {
- totalCount
- }
- streamInfo {
- dashUrl
- hlsUrl
- url
- }
- tags {
- tag {
- name
- }
- }
- restrictedReason
- id''', track_id, username, slug)
- if not cloudcast:
- raise ExtractorError('Track not found', expected=True)
- reason = cloudcast.get('restrictedReason')
- if reason == 'tracklist':
- raise ExtractorError('Track unavailable in your country due to licensing restrictions', expected=True)
- elif reason == 'repeat_play':
- raise ExtractorError('You have reached your play limit for this track', expected=True)
- elif reason:
- raise ExtractorError('Track is restricted', expected=True)
- title = cloudcast['name']
- stream_info = cloudcast['streamInfo']
- formats = []
- for url_key in ('url', 'hlsUrl', 'dashUrl'):
- format_url = stream_info.get(url_key)
- if not format_url:
- continue
- decrypted = self._decrypt_xor_cipher(
- self._DECRYPTION_KEY, base64.b64decode(format_url))
- if url_key == 'hlsUrl':
- formats.extend(self._extract_m3u8_formats(
- decrypted, track_id, 'mp4', entry_protocol='m3u8_native',
- m3u8_id='hls', fatal=False))
- elif url_key == 'dashUrl':
- formats.extend(self._extract_mpd_formats(
- decrypted, track_id, mpd_id='dash', fatal=False))
- else:
- formats.append({
- 'format_id': 'http',
- 'url': decrypted,
- 'vcodec': 'none',
- 'downloader_options': {
- # Mixcloud starts throttling at >~5M
- 'http_chunk_size': 5242880,
- },
- })
- if not formats and cloudcast.get('isExclusive'):
- self.raise_login_required(metadata_available=True)
- comments = []
- for edge in (try_get(cloudcast, lambda x: x['comments']['edges']) or []):
- node = edge.get('node') or {}
- text = strip_or_none(node.get('comment'))
- if not text:
- continue
- user = node.get('user') or {}
- comments.append({
- 'author': user.get('displayName'),
- 'author_id': user.get('username'),
- 'text': text,
- 'timestamp': parse_iso8601(node.get('created')),
- })
- tags = []
- for t in cloudcast.get('tags'):
- tag = try_get(t, lambda x: x['tag']['name'], str)
- if not tag:
- tags.append(tag)
- get_count = lambda x: int_or_none(try_get(cloudcast, lambda y: y[x]['totalCount']))
- owner = cloudcast.get('owner') or {}
- return {
- 'id': track_id,
- 'title': title,
- 'formats': formats,
- 'description': cloudcast.get('description'),
- 'thumbnail': try_get(cloudcast, lambda x: x['picture']['url'], str),
- 'uploader': owner.get('displayName'),
- 'timestamp': parse_iso8601(cloudcast.get('publishDate')),
- 'uploader_id': owner.get('username'),
- 'uploader_url': owner.get('url'),
- 'duration': int_or_none(cloudcast.get('audioLength')),
- 'view_count': int_or_none(cloudcast.get('plays')),
- 'like_count': get_count('favorites'),
- 'repost_count': get_count('reposts'),
- 'comment_count': get_count('comments'),
- 'comments': comments,
- 'tags': tags,
- 'artist': ', '.join(cloudcast.get('featuringArtistList') or []) or None,
- }
- class MixcloudPlaylistBaseIE(MixcloudBaseIE):
- def _get_cloudcast(self, node):
- return node
- def _get_playlist_title(self, title, slug):
- return title
- def _real_extract(self, url):
- username, slug = self._match_valid_url(url).groups()
- username = urllib.parse.unquote(username)
- if not slug:
- slug = 'uploads'
- else:
- slug = urllib.parse.unquote(slug)
- playlist_id = f'{username}_{slug}'
- is_playlist_type = self._ROOT_TYPE == 'playlist'
- playlist_type = 'items' if is_playlist_type else slug
- list_filter = ''
- has_next_page = True
- entries = []
- while has_next_page:
- playlist = self._call_api(
- self._ROOT_TYPE, '''%s
- %s
- %s(first: 100%s) {
- edges {
- node {
- %s
- }
- }
- pageInfo {
- endCursor
- hasNextPage
- }
- }''' % (self._TITLE_KEY, self._DESCRIPTION_KEY, playlist_type, list_filter, self._NODE_TEMPLATE), # noqa: UP031
- playlist_id, username, slug if is_playlist_type else None)
- items = playlist.get(playlist_type) or {}
- for edge in items.get('edges', []):
- cloudcast = self._get_cloudcast(edge.get('node') or {})
- cloudcast_url = cloudcast.get('url')
- if not cloudcast_url:
- continue
- item_slug = try_get(cloudcast, lambda x: x['slug'], str)
- owner_username = try_get(cloudcast, lambda x: x['owner']['username'], str)
- video_id = f'{owner_username}_{item_slug}' if item_slug and owner_username else None
- entries.append(self.url_result(
- cloudcast_url, MixcloudIE.ie_key(), video_id))
- page_info = items['pageInfo']
- has_next_page = page_info['hasNextPage']
- list_filter = ', after: "{}"'.format(page_info['endCursor'])
- return self.playlist_result(
- entries, playlist_id,
- self._get_playlist_title(playlist[self._TITLE_KEY], slug),
- playlist.get(self._DESCRIPTION_KEY))
- class MixcloudUserIE(MixcloudPlaylistBaseIE):
- _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<id>[^/]+)/(?P<type>uploads|favorites|listens|stream)?/?$'
- IE_NAME = 'mixcloud:user'
- _TESTS = [{
- 'url': 'http://www.mixcloud.com/dholbach/',
- 'info_dict': {
- 'id': 'dholbach_uploads',
- 'title': 'Daniel Holbach (uploads)',
- 'description': 'md5:a3f468a60ac8c3e1f8616380fc469b2b',
- },
- 'playlist_mincount': 36,
- }, {
- 'url': 'http://www.mixcloud.com/dholbach/uploads/',
- 'info_dict': {
- 'id': 'dholbach_uploads',
- 'title': 'Daniel Holbach (uploads)',
- 'description': 'md5:a3f468a60ac8c3e1f8616380fc469b2b',
- },
- 'playlist_mincount': 36,
- }, {
- 'url': 'http://www.mixcloud.com/dholbach/favorites/',
- 'info_dict': {
- 'id': 'dholbach_favorites',
- 'title': 'Daniel Holbach (favorites)',
- 'description': 'md5:a3f468a60ac8c3e1f8616380fc469b2b',
- },
- # 'params': {
- # 'playlist_items': '1-100',
- # },
- 'playlist_mincount': 396,
- }, {
- 'url': 'http://www.mixcloud.com/dholbach/listens/',
- 'info_dict': {
- 'id': 'dholbach_listens',
- 'title': 'Daniel Holbach (listens)',
- 'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
- },
- # 'params': {
- # 'playlist_items': '1-100',
- # },
- 'playlist_mincount': 1623,
- 'skip': 'Large list',
- }, {
- 'url': 'https://www.mixcloud.com/FirstEar/stream/',
- 'info_dict': {
- 'id': 'FirstEar_stream',
- 'title': 'First Ear (stream)',
- 'description': 'we maraud for ears',
- },
- 'playlist_mincount': 269,
- }]
- _TITLE_KEY = 'displayName'
- _DESCRIPTION_KEY = 'biog'
- _ROOT_TYPE = 'user'
- _NODE_TEMPLATE = '''slug
- url
- owner { username }'''
- def _get_playlist_title(self, title, slug):
- return f'{title} ({slug})'
- class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
- _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/playlists/(?P<playlist>[^/]+)/?$'
- IE_NAME = 'mixcloud:playlist'
- _TESTS = [{
- 'url': 'https://www.mixcloud.com/maxvibes/playlists/jazzcat-on-ness-radio/',
- 'info_dict': {
- 'id': 'maxvibes_jazzcat-on-ness-radio',
- 'title': 'Ness Radio sessions',
- },
- 'playlist_mincount': 59,
- }]
- _TITLE_KEY = 'name'
- _DESCRIPTION_KEY = 'description'
- _ROOT_TYPE = 'playlist'
- _NODE_TEMPLATE = '''cloudcast {
- slug
- url
- owner { username }
- }'''
- def _get_cloudcast(self, node):
- return node.get('cloudcast') or {}
|