[cleanup] Misc fixes

Closes #3565, https://github.com/yt-dlp/yt-dlp/issues/3514#issuecomment-1105944364

devscripts/lazy_load_template.py

@@ -7,7 +7,7 @@ class LazyLoadMetaClass(type):
     def __getattr__(cls, name):
         if '_real_class' not in cls.__dict__:
             write_string(
-                f'WARNING: Falling back to normal extractor since lazy extractor '
+                'WARNING: Falling back to normal extractor since lazy extractor '
                 f'{cls.__name__} does not have attribute {name}{bug_reports_message()}')
         return getattr(cls._get_real_class(), name)
 

yt_dlp/YoutubeDL.py

@@ -62,6 +62,7 @@ from .utils import (
     DEFAULT_OUTTMPL,
     LINK_TEMPLATES,
     NO_DEFAULT,
+    NUMBER_RE,
     OUTTMPL_TYPES,
     POSTPROCESS_WHEN,
     STR_FORMAT_RE_TMPL,
@@ -1049,7 +1050,7 @@ class YoutubeDL:
             formatSeconds(info_dict['duration'], '-' if sanitize else ':')
             if info_dict.get('duration', None) is not None
             else None)
-        info_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
+        info_dict['autonumber'] = int(self.params.get('autonumber_start', 1) - 1 + self._num_downloads)
         info_dict['video_autonumber'] = self._num_videos
         if info_dict.get('resolution') is None:
             info_dict['resolution'] = self.format_resolution(info_dict, default=None)
@@ -1071,18 +1072,18 @@ class YoutubeDL:
         # Field is of the form key1.key2...
         # where keys (except first) can be string, int or slice
         FIELD_RE = r'\w*(?:\.(?:\w+|{num}|{num}?(?::{num}?){{1,2}}))*'.format(num=r'(?:-?\d+)')
-        MATH_FIELD_RE = r'''(?:{field}|{num})'''.format(field=FIELD_RE, num=r'-?\d+(?:.\d+)?')
+        MATH_FIELD_RE = rf'(?:{FIELD_RE}|-?{NUMBER_RE})'
         MATH_OPERATORS_RE = r'(?:%s)' % '|'.join(map(re.escape, MATH_FUNCTIONS.keys()))
-        INTERNAL_FORMAT_RE = re.compile(r'''(?x)
+        INTERNAL_FORMAT_RE = re.compile(rf'''(?x)
             (?P<negate>-)?
-            (?P<fields>{field})
-            (?P<maths>(?:{math_op}{math_field})*)
+            (?P<fields>{FIELD_RE})
+            (?P<maths>(?:{MATH_OPERATORS_RE}{MATH_FIELD_RE})*)
             (?:>(?P<strf_format>.+?))?
             (?P<remaining>
                 (?P<alternate>(?<!\\),[^|&)]+)?
                 (?:&(?P<replacement>.*?))?
                 (?:\|(?P<default>.*?))?
-            )$'''.format(field=FIELD_RE, math_op=MATH_OPERATORS_RE, math_field=MATH_FIELD_RE))
+            )$''')
 
         def _traverse_infodict(k):
             k = k.split('.')
@@ -2336,7 +2337,7 @@ class YoutubeDL:
                                      video_id=info_dict['id'], ie=info_dict['extractor'])
             elif not info_dict.get('title'):
                 self.report_warning('Extractor failed to obtain "title". Creating a generic title instead')
-                info_dict['title'] = f'{info_dict["extractor"]} video #{info_dict["id"]}'
+                info_dict['title'] = f'{info_dict["extractor"].replace(":", "-")} video #{info_dict["id"]}'
 
         if info_dict.get('duration') is not None:
             info_dict['duration_string'] = formatSeconds(info_dict['duration'])
@@ -3669,10 +3670,11 @@ class YoutubeDL:
         ) or 'none'
         write_debug('exe versions: %s' % exe_str)
 
+        from .compat.compat_utils import get_package_info
         from .dependencies import available_dependencies
 
         write_debug('Optional libraries: %s' % (', '.join(sorted({
-            module.__name__.split('.')[0] for module in available_dependencies.values()
+            join_nonempty(*get_package_info(m)) for m in available_dependencies.values()
         })) or 'none'))
 
         self._setup_opener()

yt_dlp/compat/__init__.py

@@ -46,10 +46,6 @@ def compat_ord(c):
     return c if isinstance(c, int) else ord(c)
 
 
-def compat_setenv(key, value, env=os.environ):
-    env[key] = value
-
-
 if compat_os_name == 'nt' and sys.version_info < (3, 8):
     # os.path.realpath on Windows does not follow symbolic links
     # prior to Python 3.8 (see https://bugs.python.org/issue9949)

yt_dlp/compat/_deprecated.py

@@ -44,4 +44,9 @@ compat_urllib_parse_urlparse = urllib.parse.urlparse
 compat_urllib_request = urllib.request
 compat_urlparse = compat_urllib_parse = urllib.parse
 
+
+def compat_setenv(key, value, env=os.environ):
+    env[key] = value
+
+
 __all__ = [x for x in globals() if x.startswith('compat_')]

yt_dlp/compat/asyncio.py

@@ -1,5 +1,4 @@
 # flake8: noqa: F405
-
 from asyncio import *  # noqa: F403
 
 from .compat_utils import passthrough_module

yt_dlp/compat/compat_utils.py

@@ -1,9 +1,28 @@
+import collections
 import contextlib
 import importlib
 import sys
 import types
 
 
+_NO_ATTRIBUTE = object()
+
+_Package = collections.namedtuple('Package', ('name', 'version'))
+
+
+def get_package_info(module):
+    parent = module.__name__.split('.')[0]
+    parent_module = None
+    with contextlib.suppress(ImportError):
+        parent_module = importlib.import_module(parent)
+
+    for attr in ('__version__', 'version_string', 'version'):
+        version = getattr(parent_module, attr, None)
+        if version is not None:
+            break
+    return _Package(getattr(module, '_yt_dlp__identifier', parent), str(version))
+
+
 def _is_package(module):
     try:
         module.__getattribute__('__path__')
@@ -12,9 +31,6 @@ def _is_package(module):
     return True
 
 
-_NO_ATTRIBUTE = object()
-
-
 def passthrough_module(parent, child, *, callback=lambda _: None):
     parent_module = importlib.import_module(parent)
     child_module = importlib.import_module(child, parent)

yt_dlp/compat/re.py

@@ -1,5 +1,4 @@
 # flake8: noqa: F405
-
 from re import *  # F403
 
 from .compat_utils import passthrough_module

yt_dlp/dependencies.py

@@ -1,4 +1,6 @@
 # flake8: noqa: F401
+"""Imports all optional dependencies for the project.
+An attribute "_yt_dlp__identifier" may be inserted into the module if it uses an ambigious namespace"""
 
 try:
     import brotlicffi as brotli
@@ -28,6 +30,15 @@ except ImportError:
         from Crypto.Cipher import AES as Cryptodome_AES
     except ImportError:
         Cryptodome_AES = None
+    else:
+        try:
+            # In pycrypto, mode defaults to ECB. See:
+            # https://www.pycryptodome.org/en/latest/src/vs_pycrypto.html#:~:text=not%20have%20ECB%20as%20default%20mode
+            Cryptodome_AES.new(b'abcdefghijklmnop')
+        except TypeError:
+            pass
+        else:
+            Cryptodome_AES._yt_dlp__identifier = 'pycrypto'
 
 
 try:

yt_dlp/downloader/common.py

@@ -12,6 +12,7 @@ from ..minicurses import (
     QuietMultilinePrinter,
 )
 from ..utils import (
+    NUMBER_RE,
     LockingUnsupportedError,
     Namespace,
     decodeArgument,
@@ -91,7 +92,8 @@ class FileDownloader:
             'trouble',
             'write_debug',
         ):
-            setattr(self, func, getattr(ydl, func))
+            if not hasattr(self, func):
+                setattr(self, func, getattr(ydl, func))
 
     def to_screen(self, *args, **kargs):
         self.ydl.to_screen(*args, quiet=self.params.get('quiet'), **kargs)
@@ -170,7 +172,7 @@ class FileDownloader:
     @staticmethod
     def parse_bytes(bytestr):
         """Parse a string indicating a byte quantity into an integer."""
-        matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
+        matchobj = re.match(rf'(?i)^({NUMBER_RE})([kMGTPEZY]?)$', bytestr)
         if matchobj is None:
             return None
         number = float(matchobj.group(1))

yt_dlp/downloader/external.py

@@ -368,7 +368,7 @@ class FFmpegFD(ExternalFD):
 
         # These exists only for compatibility. Extractors should use
         # info_dict['downloader_options']['ffmpeg_args'] instead
-        args += info_dict.get('_ffmpeg_args')
+        args += info_dict.get('_ffmpeg_args') or []
         seekable = info_dict.get('_seekable')
         if seekable is not None:
             # setting -seekable prevents ffmpeg from guessing if the server