Simplify tests

* Make them directly executable again
* Move common stuff (md5, parameters) to helper
* Never import *
* General clean up

test/helper.py

@@ -1,22 +1,27 @@
 import errno
 import io
+import hashlib
 import json
 import os.path
 import re
 import types
 
 import youtube_dl.extractor
-from youtube_dl import YoutubeDL, YoutubeDLHandler
-from youtube_dl.utils import (
-    compat_cookiejar,
-    compat_urllib_request,
-)
+from youtube_dl import YoutubeDL
 
-youtube_dl._setup_opener(timeout=10)
 
-PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
-with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
-    parameters = json.load(pf)
+def global_setup():
+    youtube_dl._setup_opener(timeout=10)
+
+
+def get_params(override=None):
+    PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                   "parameters.json")
+    with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+        parameters = json.load(pf)
+    if override:
+        parameters.update(override)
+    return parameters
 
 
 def try_rm(filename):
@@ -32,7 +37,7 @@ class FakeYDL(YoutubeDL):
     def __init__(self):
         # Different instances of the downloader can't share the same dictionary
         # some test set the "sublang" parameter, which would break the md5 checks.
-        params = dict(parameters)
+        params = get_params()
         super(FakeYDL, self).__init__(params)
         self.result = []
         
@@ -62,3 +67,6 @@ def get_testcases():
         for t in getattr(ie, '_TESTS', []):
             t['name'] = type(ie).__name__[:-len('IE')]
             yield t
+
+
+md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()

test/test_age_restriction.py

@@ -1,14 +1,16 @@
 #!/usr/bin/env python
 
+# Allow direct execution
+import os
 import sys
 import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import global_setup, try_rm
+global_setup()
 
-# Allow direct execution
-import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from youtube_dl import YoutubeDL
-from .helper import try_rm
 
 
 def _download_restricted(url, filename, age):

test/test_all_urls.py

@@ -1,14 +1,20 @@
 #!/usr/bin/env python
 
+# Allow direct execution
+import os
 import sys
 import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-# Allow direct execution
-import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from youtube_dl.extractor import YoutubeIE, YoutubePlaylistIE, YoutubeChannelIE, JustinTVIE, gen_extractors
-from .helper import get_testcases
+from test.helper import get_testcases
+
+from youtube_dl.extractor import (
+    gen_extractors,
+    JustinTVIE,
+    YoutubeIE,
+)
+
 
 class TestAllURLsMatching(unittest.TestCase):
     def setUp(self):

test/test_dailymotion_subtitles.py

@@ -1,18 +1,16 @@
 #!/usr/bin/env python
 
+# Allow direct execution
+import os
 import sys
 import unittest
-import hashlib
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-# Allow direct execution
-import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from test.helper import FakeYDL, global_setup, md5
+global_setup()
 
-from youtube_dl.extractor import DailymotionIE
-from youtube_dl.utils import *
-from .helper import FakeYDL
 
-md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
+from youtube_dl.extractor import DailymotionIE
 
 class TestDailymotionSubtitles(unittest.TestCase):
     def setUp(self):

test/test_download.py

@@ -1,26 +1,31 @@
 #!/usr/bin/env python
 
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import get_params, get_testcases, global_setup, try_rm, md5
+global_setup()
+
+
 import hashlib
 import io
-import os
 import json
-import unittest
-import sys
 import socket
-import binascii
-
-# Allow direct execution
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import youtube_dl.YoutubeDL
-from youtube_dl.utils import *
-
-PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+from youtube_dl.utils import (
+    compat_str,
+    compat_urllib_error,
+    DownloadError,
+    ExtractorError,
+    UnavailableVideoError,
+)
 
 RETRIES = 3
 
-md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
-
 class YoutubeDL(youtube_dl.YoutubeDL):
     def __init__(self, *args, **kwargs):
         self.to_stderr = self.to_screen
@@ -37,18 +42,12 @@ def _file_md5(fn):
     with open(fn, 'rb') as f:
         return hashlib.md5(f.read()).hexdigest()
 
-import test.helper as helper  # Set up remaining global configuration
-from .helper import get_testcases, try_rm
 defs = get_testcases()
 
-with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
-    parameters = json.load(pf)
-
 
 class TestDownload(unittest.TestCase):
     maxDiff = None
     def setUp(self):
-        self.parameters = parameters
         self.defs = defs
 
 ### Dynamically generate tests
@@ -68,8 +67,7 @@ def generator(test_case):
             print_skipping(test_case['skip'])
             return
 
-        params = self.parameters.copy()
-        params.update(test_case.get('params', {}))
+        params = get_params(test_case.get('params', {}))
 
         ydl = YoutubeDL(params)
         ydl.add_default_info_extractors()

test/test_playlists.py

@@ -1,13 +1,16 @@
 #!/usr/bin/env python
 # encoding: utf-8
 
-import sys
-import unittest
-import json
 
 # Allow direct execution
 import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import FakeYDL, global_setup
+global_setup()
+
 
 from youtube_dl.extractor import (
     DailymotionPlaylistIE,
@@ -18,9 +21,7 @@ from youtube_dl.extractor import (
     LivestreamIE,
     NHLVideocenterIE,
 )
-from youtube_dl.utils import *
 
-from .helper import FakeYDL
 
 class TestPlaylists(unittest.TestCase):
     def assertIsPlaylist(self, info):

test/test_utils.py

@@ -1,14 +1,14 @@
 #!/usr/bin/env python
 
-# Various small unit tests
-
+# Allow direct execution
+import os
 import sys
 import unittest
-import xml.etree.ElementTree
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-# Allow direct execution
-import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+# Various small unit tests
+import xml.etree.ElementTree
 
 #from youtube_dl.utils import htmlentity_transform
 from youtube_dl.utils import (

test/test_write_annotations.py

@@ -1,39 +1,38 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-import xml.etree.ElementTree
+# Allow direct execution
 import os
 import sys
 import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-# Allow direct execution
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from test.helper import get_params, global_setup, try_rm
+global_setup()
+
+
+import io
+
+import xml.etree.ElementTree
 
 import youtube_dl.YoutubeDL
 import youtube_dl.extractor
-from youtube_dl.utils import *
-from .helper import try_rm
-
-PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+from youtube_dl.utils import True
 
-# General configuration (from __init__, not very elegant...)
-jar = compat_cookiejar.CookieJar()
-cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
-proxy_handler = compat_urllib_request.ProxyHandler()
-opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
-compat_urllib_request.install_opener(opener)
 
 class YoutubeDL(youtube_dl.YoutubeDL):
     def __init__(self, *args, **kwargs):
         super(YoutubeDL, self).__init__(*args, **kwargs)
         self.to_stderr = self.to_screen
 
-with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
-    params = json.load(pf)
-params['writeannotations'] = True
-params['skip_download'] = True
-params['writeinfojson'] = False
-params['format'] = 'flv'
+params = get_params({
+    'writeannotations': True,
+    'skip_download': True,
+    'writeinfojson': False,
+    'format': 'flv',
+})
+
+
 
 TEST_ID = 'gr51aVj-mLg'
 ANNOTATIONS_FILE = TEST_ID + '.flv.annotations.xml'

test/test_write_info_json.py

@@ -1,37 +1,34 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-import json
+# Allow direct execution
 import os
 import sys
 import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-# Allow direct execution
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from test.helper import get_params, global_setup
+global_setup()
+
+
+import io
+import json
 
 import youtube_dl.YoutubeDL
 import youtube_dl.extractor
-from youtube_dl.utils import *
-
-PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
 
-# General configuration (from __init__, not very elegant...)
-jar = compat_cookiejar.CookieJar()
-cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
-proxy_handler = compat_urllib_request.ProxyHandler()
-opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
-compat_urllib_request.install_opener(opener)
 
 class YoutubeDL(youtube_dl.YoutubeDL):
     def __init__(self, *args, **kwargs):
         super(YoutubeDL, self).__init__(*args, **kwargs)
         self.to_stderr = self.to_screen
 
-with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
-    params = json.load(pf)
-params['writeinfojson'] = True
-params['skip_download'] = True
-params['writedescription'] = True
+params = get_params({
+    'writeinfojson': True,
+    'skip_download': True,
+    'writedescription': True,
+})
+
 
 TEST_ID = 'BaW_jenozKc'
 INFO_JSON_FILE = TEST_ID + '.mp4.info.json'
@@ -42,6 +39,7 @@ This is a test video for youtube-dl.
 
 For more information, contact phihag@phihag.de .'''
 
+
 class TestInfoJSON(unittest.TestCase):
     def setUp(self):
         # Clear old files

test/test_youtube_lists.py

@@ -1,20 +1,26 @@
 #!/usr/bin/env python
 
+# Allow direct execution
+import os
 import sys
 import unittest
-import json
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import FakeYDL, global_setup
+global_setup()
 
-# Allow direct execution
-import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from youtube_dl.extractor import YoutubeUserIE, YoutubePlaylistIE, YoutubeIE, YoutubeChannelIE, YoutubeShowIE
-from youtube_dl.utils import *
+from youtube_dl.extractor import (
+    YoutubeUserIE,
+    YoutubePlaylistIE,
+    YoutubeIE,
+    YoutubeChannelIE,
+    YoutubeShowIE,
+)
 
-from .helper import FakeYDL
 
 class TestYoutubeLists(unittest.TestCase):
-    def assertIsPlaylist(self,info):
+    def assertIsPlaylist(self, info):
         """Make sure the info has '_type' set to 'playlist'"""
         self.assertEqual(info['_type'], 'playlist')