|
|
@@ -0,0 +1,349 @@
|
|
|
+#!/usr/bin/env python
|
|
|
+
|
|
|
+# Please, keep this script in sync with arcadia/ya
|
|
|
+
|
|
|
+import os
|
|
|
+import sys
|
|
|
+import platform
|
|
|
+
|
|
|
+RETRIES = 5
|
|
|
+HASH_PREFIX = 10
|
|
|
+
|
|
|
+REGISTRY_ENDPOINT = os.environ.get("YA_REGISTRY_ENDPOINT", "https://s3.mds.yandex.net/devtools-registry")
|
|
|
+
|
|
|
+
|
|
|
+PLATFORM_MAP = {
|
|
|
+ "data": {
|
|
|
+ "darwin": {
|
|
|
+ "md5": "1356748b6bb7ce19359bc1c87d81cf5e",
|
|
|
+ "urls": [
|
|
|
+ f"{REGISTRY_ENDPOINT}/4948880410",
|
|
|
+ ],
|
|
|
+ },
|
|
|
+ "darwin-arm64": {
|
|
|
+ "md5": "2ba999e77660102f78c2584931eef3a2",
|
|
|
+ "urls": [
|
|
|
+ f"{REGISTRY_ENDPOINT}/4948879923",
|
|
|
+ ],
|
|
|
+ },
|
|
|
+ "linux-aarch64": {
|
|
|
+ "md5": "731df682969dddc89bd7f3160960ffa1",
|
|
|
+ "urls": [
|
|
|
+ f"{REGISTRY_ENDPOINT}/4948879460",
|
|
|
+ ],
|
|
|
+ },
|
|
|
+ "win32-clang-cl": {
|
|
|
+ "md5": "e2289cac60bf800470d858af48fce490",
|
|
|
+ "urls": [
|
|
|
+ f"{REGISTRY_ENDPOINT}/4948880870",
|
|
|
+ ],
|
|
|
+ },
|
|
|
+ "linux": {
|
|
|
+ "md5": "c71678ed371ae966d7c0235352a398e3",
|
|
|
+ "urls": [
|
|
|
+ f"{REGISTRY_ENDPOINT}/4948881174",
|
|
|
+ ],
|
|
|
+ },
|
|
|
+ }
|
|
|
+}
|
|
|
+# End of mapping
|
|
|
+
|
|
|
+
|
|
|
+def create_dirs(path):
|
|
|
+ try:
|
|
|
+ os.makedirs(path)
|
|
|
+ except OSError as e:
|
|
|
+ import errno
|
|
|
+
|
|
|
+ if e.errno != errno.EEXIST:
|
|
|
+ raise
|
|
|
+
|
|
|
+ return path
|
|
|
+
|
|
|
+
|
|
|
+def home_dir():
|
|
|
+ # Do not trust $HOME, as it is unreliable in certain environments
|
|
|
+ # Temporarily delete os.environ["HOME"] to force reading current home directory from /etc/passwd
|
|
|
+ home_from_env = os.environ.pop("HOME", None)
|
|
|
+ try:
|
|
|
+ home_from_passwd = os.path.expanduser("~")
|
|
|
+ if os.path.isabs(home_from_passwd):
|
|
|
+ # This home dir is valid, prefer it over $HOME
|
|
|
+ return home_from_passwd
|
|
|
+ else:
|
|
|
+ # When python is built with musl (this is quire weird though),
|
|
|
+ # only users from /etc/passwd will be properly resolved,
|
|
|
+ # as musl does not have nss module for LDAP integration.
|
|
|
+ return home_from_env
|
|
|
+
|
|
|
+ finally:
|
|
|
+ if home_from_env is not None:
|
|
|
+ os.environ["HOME"] = home_from_env
|
|
|
+
|
|
|
+
|
|
|
+def misc_root():
|
|
|
+ return create_dirs(os.getenv('YA_CACHE_DIR') or os.path.join(home_dir(), '.ya'))
|
|
|
+
|
|
|
+
|
|
|
+def tool_root():
|
|
|
+ return create_dirs(os.getenv('YA_CACHE_DIR_TOOLS') or os.path.join(misc_root(), 'tools'))
|
|
|
+
|
|
|
+
|
|
|
+# TODO: remove when switched to S3, won't be needed in OSS
|
|
|
+def ya_token():
|
|
|
+ def get_token_from_file():
|
|
|
+ try:
|
|
|
+ with open(os.environ.get('YA_TOKEN_PATH', os.path.join(home_dir(), '.ya_token')), 'r') as f:
|
|
|
+ return f.read().strip()
|
|
|
+ except:
|
|
|
+ pass
|
|
|
+
|
|
|
+ return os.getenv('YA_TOKEN') or get_token_from_file()
|
|
|
+
|
|
|
+
|
|
|
+TOOLS_DIR = tool_root()
|
|
|
+
|
|
|
+
|
|
|
+def uniq(size=6):
|
|
|
+ import string
|
|
|
+ import random
|
|
|
+
|
|
|
+ return ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(size))
|
|
|
+
|
|
|
+
|
|
|
+_ssl_is_tuned = False
|
|
|
+
|
|
|
+
|
|
|
+def _tune_ssl():
|
|
|
+ global _ssl_is_tuned
|
|
|
+ if _ssl_is_tuned:
|
|
|
+ return
|
|
|
+
|
|
|
+ try:
|
|
|
+ import ssl
|
|
|
+
|
|
|
+ ssl._create_default_https_context = ssl._create_unverified_context
|
|
|
+ except AttributeError:
|
|
|
+ pass
|
|
|
+
|
|
|
+ try:
|
|
|
+ import urllib3
|
|
|
+
|
|
|
+ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
|
+ except (AttributeError, ImportError):
|
|
|
+ pass
|
|
|
+ _ssl_is_tuned = True
|
|
|
+
|
|
|
+
|
|
|
+def _fetch(url, into):
|
|
|
+ import hashlib
|
|
|
+
|
|
|
+ _tune_ssl()
|
|
|
+
|
|
|
+ from urllib.request import urlopen
|
|
|
+ from urllib.request import Request
|
|
|
+ from urllib.parse import urlparse
|
|
|
+
|
|
|
+ request = Request(str(url))
|
|
|
+ # TODO: Remove when switched to S3 distribution
|
|
|
+ request.add_header('User-Agent', 'ya-bootstrap')
|
|
|
+ token = ya_token()
|
|
|
+ if token:
|
|
|
+ request.add_header('Authorization', 'OAuth {}'.format(token))
|
|
|
+
|
|
|
+ md5 = hashlib.md5()
|
|
|
+ sys.stderr.write('Downloading %s ' % url)
|
|
|
+ sys.stderr.flush()
|
|
|
+ conn = urlopen(request, timeout=10)
|
|
|
+ sys.stderr.write('[')
|
|
|
+ sys.stderr.flush()
|
|
|
+ try:
|
|
|
+ with open(into, 'wb') as f:
|
|
|
+ while True:
|
|
|
+ block = conn.read(1024 * 1024)
|
|
|
+ sys.stderr.write('.')
|
|
|
+ sys.stderr.flush()
|
|
|
+ if block:
|
|
|
+ md5.update(block)
|
|
|
+ f.write(block)
|
|
|
+ else:
|
|
|
+ break
|
|
|
+ return md5.hexdigest()
|
|
|
+
|
|
|
+ finally:
|
|
|
+ sys.stderr.write('] ')
|
|
|
+ sys.stderr.flush()
|
|
|
+
|
|
|
+
|
|
|
+def _atomic_fetch(url, into, md5):
|
|
|
+ tmp_dest = into + '.' + uniq()
|
|
|
+ try:
|
|
|
+ real_md5 = _fetch(url, tmp_dest)
|
|
|
+ if real_md5 != md5:
|
|
|
+ raise Exception('MD5 mismatched: %s differs from %s' % (real_md5, md5))
|
|
|
+ os.rename(tmp_dest, into)
|
|
|
+ sys.stderr.write('OK\n')
|
|
|
+ except Exception as e:
|
|
|
+ sys.stderr.write('ERROR: ' + str(e) + '\n')
|
|
|
+ raise
|
|
|
+ finally:
|
|
|
+ try:
|
|
|
+ os.remove(tmp_dest)
|
|
|
+ except OSError:
|
|
|
+ pass
|
|
|
+
|
|
|
+
|
|
|
+def _extract(path, into):
|
|
|
+ import tarfile
|
|
|
+
|
|
|
+ tar = tarfile.open(path, errorlevel=2)
|
|
|
+
|
|
|
+ # tar.extractall() will try to set file ownership according to the attributes stored in the archive
|
|
|
+ # by calling TarFile.chown() method.
|
|
|
+ # As this information is hardly relevant to the point of deployment / extraction,
|
|
|
+ # it will just fail (python2) if ya is executed with root euid, or silently set non-existent numeric owner (python3)
|
|
|
+ # to the files being extracted.
|
|
|
+ # mock it with noop to retain current user ownership.
|
|
|
+ tar.chown = lambda *args, **kwargs: None
|
|
|
+
|
|
|
+ tar.extractall(path=into)
|
|
|
+ tar.close()
|
|
|
+
|
|
|
+
|
|
|
+def _get(urls, md5):
|
|
|
+ dest_path = os.path.join(TOOLS_DIR, md5[:HASH_PREFIX])
|
|
|
+
|
|
|
+ if not os.path.exists(dest_path):
|
|
|
+ for iter in range(RETRIES):
|
|
|
+ try:
|
|
|
+ _atomic_fetch(urls[iter % len(urls)], dest_path, md5)
|
|
|
+ break
|
|
|
+ except Exception:
|
|
|
+ if iter + 1 == RETRIES:
|
|
|
+ raise
|
|
|
+ else:
|
|
|
+ import time
|
|
|
+
|
|
|
+ time.sleep(iter)
|
|
|
+
|
|
|
+ return dest_path
|
|
|
+
|
|
|
+
|
|
|
+def _get_dir(urls, md5, ya_name):
|
|
|
+ dest_dir = os.path.join(TOOLS_DIR, md5[:HASH_PREFIX] + '_d')
|
|
|
+
|
|
|
+ if os.path.isfile(os.path.join(dest_dir, ya_name)):
|
|
|
+ return dest_dir
|
|
|
+
|
|
|
+ try:
|
|
|
+ packed_path = _get(urls, md5)
|
|
|
+ except Exception:
|
|
|
+ if os.path.isfile(os.path.join(dest_dir, ya_name)):
|
|
|
+ return dest_dir
|
|
|
+ raise
|
|
|
+
|
|
|
+ tmp_dir = dest_dir + '.' + uniq()
|
|
|
+ try:
|
|
|
+ try:
|
|
|
+ _extract(packed_path, tmp_dir)
|
|
|
+ except Exception:
|
|
|
+ if os.path.isfile(os.path.join(dest_dir, ya_name)):
|
|
|
+ return dest_dir
|
|
|
+ raise
|
|
|
+
|
|
|
+ try:
|
|
|
+ os.rename(tmp_dir, dest_dir)
|
|
|
+ except OSError as e:
|
|
|
+ import errno
|
|
|
+
|
|
|
+ if e.errno != errno.ENOTEMPTY:
|
|
|
+ raise
|
|
|
+
|
|
|
+ return dest_dir
|
|
|
+ finally:
|
|
|
+ import shutil
|
|
|
+
|
|
|
+ shutil.rmtree(tmp_dir, ignore_errors=True)
|
|
|
+ try:
|
|
|
+ os.remove(packed_path)
|
|
|
+ except Exception:
|
|
|
+ pass
|
|
|
+
|
|
|
+
|
|
|
+def _mine_repo_root():
|
|
|
+ # We think that this script is located in the root of the repo.
|
|
|
+ return os.path.dirname(os.path.realpath(__file__))
|
|
|
+
|
|
|
+
|
|
|
+def main():
|
|
|
+ if not os.path.exists(TOOLS_DIR):
|
|
|
+ os.makedirs(TOOLS_DIR)
|
|
|
+
|
|
|
+ result_args = sys.argv[1:]
|
|
|
+
|
|
|
+ meta = PLATFORM_MAP['data']
|
|
|
+ my_platform = platform.system().lower()
|
|
|
+ my_machine = platform.machine().lower()
|
|
|
+ if my_platform == 'linux':
|
|
|
+ if 'ppc64le' in platform.platform():
|
|
|
+ my_platform = 'linux-ppc64le'
|
|
|
+ elif 'aarch64' in platform.platform():
|
|
|
+ my_platform = 'linux-aarch64'
|
|
|
+ else:
|
|
|
+ my_platform = 'linux_musl'
|
|
|
+ if my_platform == 'darwin' and my_machine == 'arm64':
|
|
|
+ my_platform = 'darwin-arm64'
|
|
|
+
|
|
|
+ def _platform_key(target_platform):
|
|
|
+ """match by max prefix length, prefer shortest"""
|
|
|
+
|
|
|
+ def _key_for_platform(platform):
|
|
|
+ return len(os.path.commonprefix([target_platform, platform])), -len(platform)
|
|
|
+
|
|
|
+ return _key_for_platform
|
|
|
+
|
|
|
+ best_key = max(meta.keys(), key=_platform_key(my_platform))
|
|
|
+ value = meta[best_key]
|
|
|
+
|
|
|
+ ya_name = {'win32': 'ya-bin.exe', 'win32-clang-cl': 'ya-bin.exe'}.get(best_key, 'ya-bin') # XXX
|
|
|
+ ya_dir = _get_dir(value['urls'], value['md5'], ya_name)
|
|
|
+
|
|
|
+ # Popen `args` must have `str` type
|
|
|
+ ya_path = str(os.path.join(ya_dir, ya_name))
|
|
|
+
|
|
|
+ env = os.environ.copy()
|
|
|
+ if 'YA_SOURCE_ROOT' not in env:
|
|
|
+ src_root = _mine_repo_root()
|
|
|
+ if src_root is not None:
|
|
|
+ env['YA_SOURCE_ROOT'] = src_root
|
|
|
+
|
|
|
+ for env_name in [
|
|
|
+ 'LD_PRELOAD',
|
|
|
+ 'Y_PYTHON_SOURCE_ROOT',
|
|
|
+ ]:
|
|
|
+ if env_name in os.environ:
|
|
|
+ sys.stderr.write(
|
|
|
+ "Warn: {}='{}' is specified and may affect the correct operation of the ya\n".format(
|
|
|
+ env_name, env[env_name]
|
|
|
+ )
|
|
|
+ )
|
|
|
+
|
|
|
+ if os.name == 'nt':
|
|
|
+ import subprocess
|
|
|
+
|
|
|
+ p = subprocess.Popen([ya_path] + result_args, env=env)
|
|
|
+ p.wait()
|
|
|
+ sys.exit(p.returncode)
|
|
|
+ else:
|
|
|
+ os.execve(ya_path, [ya_path] + result_args, env)
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == '__main__':
|
|
|
+ try:
|
|
|
+ main()
|
|
|
+ except Exception as e:
|
|
|
+ sys.stderr.write('ERROR: ' + str(e) + '\n')
|
|
|
+ from traceback import format_exc
|
|
|
+
|
|
|
+ sys.stderr.write(format_exc() + "\n")
|
|
|
+ sys.exit(1)
|
prettyboy