ya 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371
  1. #!/usr/bin/env sh
  2. # Please, keep this script in sync with arcadia/ya
  3. # Shell commands follow
  4. # Next line is bilingual: it starts a comment in Python, but do nothing in shell
  5. """:"
  6. # Find a suitable python interpreter
  7. for cmd in python3 python; do
  8. command -v > /dev/null $cmd && exec `command -v $cmd` $0 "$@"
  9. done
  10. echo "Python interpreter is not found in this system, please, install python" >&2
  11. exit 2
  12. ":"""
  13. # Previous line is bilingual: it ends a comment in Python, but do nothing in shell
  14. # Shell commands end here
  15. # Python script follows
  16. import os
  17. import sys
  18. import platform
  19. RETRIES = 5
  20. HASH_PREFIX = 10
  21. REGISTRY_ENDPOINT = os.environ.get("YA_REGISTRY_ENDPOINT", "https://devtools-registry.s3.yandex.net")
  22. # Please do not change this dict, it is updated automatically
  23. # Start of mapping
  24. PLATFORM_MAP = {
  25. "data": {
  26. "darwin": {
  27. "md5": "9eb15ad1ec74b8b1926d03630b4ed35a",
  28. "urls": [
  29. f"{REGISTRY_ENDPOINT}/5545697562"
  30. ]
  31. },
  32. "darwin-arm64": {
  33. "md5": "3e2acd9eba7df61abfef440954624d73",
  34. "urls": [
  35. f"{REGISTRY_ENDPOINT}/5545697168"
  36. ]
  37. },
  38. "linux-aarch64": {
  39. "md5": "7542537fcf90a4fb3a4fe612bcb2f251",
  40. "urls": [
  41. f"{REGISTRY_ENDPOINT}/5545696484"
  42. ]
  43. },
  44. "win32-clang-cl": {
  45. "md5": "0c380c91891731860965887af58f6fec",
  46. "urls": [
  47. f"{REGISTRY_ENDPOINT}/5545698486"
  48. ]
  49. },
  50. "linux": {
  51. "md5": "428c3f0525c9220e83a5d7cb9ab690eb",
  52. "urls": [
  53. f"{REGISTRY_ENDPOINT}/5545699155"
  54. ]
  55. }
  56. }
  57. } # End of mapping
  58. def create_dirs(path):
  59. try:
  60. os.makedirs(path)
  61. except OSError as e:
  62. import errno
  63. if e.errno != errno.EEXIST:
  64. raise
  65. return path
  66. def home_dir():
  67. # Do not trust $HOME, as it is unreliable in certain environments
  68. # Temporarily delete os.environ["HOME"] to force reading current home directory from /etc/passwd
  69. home_from_env = os.environ.pop("HOME", None)
  70. try:
  71. home_from_passwd = os.path.expanduser("~")
  72. if os.path.isabs(home_from_passwd):
  73. # This home dir is valid, prefer it over $HOME
  74. return home_from_passwd
  75. else:
  76. # When python is built with musl (this is quire weird though),
  77. # only users from /etc/passwd will be properly resolved,
  78. # as musl does not have nss module for LDAP integration.
  79. return home_from_env
  80. finally:
  81. if home_from_env is not None:
  82. os.environ["HOME"] = home_from_env
  83. def misc_root():
  84. return create_dirs(os.getenv('YA_CACHE_DIR') or os.path.join(home_dir(), '.ya'))
  85. def tool_root():
  86. return create_dirs(os.getenv('YA_CACHE_DIR_TOOLS') or os.path.join(misc_root(), 'tools'))
  87. # TODO: remove when switched to S3, won't be needed in OSS
  88. def ya_token():
  89. def get_token_from_file():
  90. try:
  91. with open(os.environ.get('YA_TOKEN_PATH', os.path.join(home_dir(), '.ya_token')), 'r') as f:
  92. return f.read().strip()
  93. except:
  94. pass
  95. return os.getenv('YA_TOKEN') or get_token_from_file()
  96. TOOLS_DIR = tool_root()
  97. def uniq(size=6):
  98. import string
  99. import random
  100. return ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(size))
  101. _ssl_is_tuned = False
  102. def _tune_ssl():
  103. global _ssl_is_tuned
  104. if _ssl_is_tuned:
  105. return
  106. try:
  107. import ssl
  108. ssl._create_default_https_context = ssl._create_unverified_context
  109. except AttributeError:
  110. pass
  111. try:
  112. import urllib3
  113. urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
  114. except (AttributeError, ImportError):
  115. pass
  116. _ssl_is_tuned = True
  117. def _fetch(url, into):
  118. import hashlib
  119. _tune_ssl()
  120. from urllib.request import urlopen
  121. from urllib.request import Request
  122. from urllib.parse import urlparse
  123. request = Request(str(url))
  124. # TODO: Remove when switched to S3 distribution
  125. request.add_header('User-Agent', 'ya-bootstrap')
  126. token = ya_token()
  127. if token:
  128. request.add_header('Authorization', 'OAuth {}'.format(token))
  129. md5 = hashlib.md5()
  130. sys.stderr.write('Downloading %s ' % url)
  131. sys.stderr.flush()
  132. conn = urlopen(request, timeout=10)
  133. sys.stderr.write('[')
  134. sys.stderr.flush()
  135. try:
  136. with open(into, 'wb') as f:
  137. while True:
  138. block = conn.read(1024 * 1024)
  139. sys.stderr.write('.')
  140. sys.stderr.flush()
  141. if block:
  142. md5.update(block)
  143. f.write(block)
  144. else:
  145. break
  146. return md5.hexdigest()
  147. finally:
  148. sys.stderr.write('] ')
  149. sys.stderr.flush()
  150. def _atomic_fetch(url, into, md5):
  151. tmp_dest = into + '.' + uniq()
  152. try:
  153. real_md5 = _fetch(url, tmp_dest)
  154. if real_md5 != md5:
  155. raise Exception('MD5 mismatched: %s differs from %s' % (real_md5, md5))
  156. os.rename(tmp_dest, into)
  157. sys.stderr.write('OK\n')
  158. except Exception as e:
  159. sys.stderr.write('ERROR: ' + str(e) + '\n')
  160. raise
  161. finally:
  162. try:
  163. os.remove(tmp_dest)
  164. except OSError:
  165. pass
  166. def _extract(path, into):
  167. import tarfile
  168. tar = tarfile.open(path, errorlevel=2)
  169. # tar.extractall() will try to set file ownership according to the attributes stored in the archive
  170. # by calling TarFile.chown() method.
  171. # As this information is hardly relevant to the point of deployment / extraction,
  172. # it will just fail (python2) if ya is executed with root euid, or silently set non-existent numeric owner (python3)
  173. # to the files being extracted.
  174. # mock it with noop to retain current user ownership.
  175. tar.chown = lambda *args, **kwargs: None
  176. tar.extractall(path=into)
  177. tar.close()
  178. def _get(urls, md5):
  179. dest_path = os.path.join(TOOLS_DIR, md5[:HASH_PREFIX])
  180. if not os.path.exists(dest_path):
  181. for iter in range(RETRIES):
  182. try:
  183. _atomic_fetch(urls[iter % len(urls)], dest_path, md5)
  184. break
  185. except Exception:
  186. if iter + 1 == RETRIES:
  187. raise
  188. else:
  189. import time
  190. time.sleep(iter)
  191. return dest_path
  192. def _get_dir(urls, md5, ya_name):
  193. dest_dir = os.path.join(TOOLS_DIR, md5[:HASH_PREFIX] + '_d')
  194. if os.path.isfile(os.path.join(dest_dir, ya_name)):
  195. return dest_dir
  196. try:
  197. packed_path = _get(urls, md5)
  198. except Exception:
  199. if os.path.isfile(os.path.join(dest_dir, ya_name)):
  200. return dest_dir
  201. raise
  202. tmp_dir = dest_dir + '.' + uniq()
  203. try:
  204. try:
  205. _extract(packed_path, tmp_dir)
  206. except Exception:
  207. if os.path.isfile(os.path.join(dest_dir, ya_name)):
  208. return dest_dir
  209. raise
  210. try:
  211. os.rename(tmp_dir, dest_dir)
  212. except OSError as e:
  213. import errno
  214. if e.errno != errno.ENOTEMPTY:
  215. raise
  216. return dest_dir
  217. finally:
  218. import shutil
  219. shutil.rmtree(tmp_dir, ignore_errors=True)
  220. try:
  221. os.remove(packed_path)
  222. except Exception:
  223. pass
  224. def _mine_repo_root():
  225. # We think that this script is located in the root of the repo.
  226. return os.path.dirname(os.path.realpath(__file__))
  227. def main():
  228. if not os.path.exists(TOOLS_DIR):
  229. os.makedirs(TOOLS_DIR)
  230. result_args = sys.argv[1:]
  231. meta = PLATFORM_MAP['data']
  232. my_platform = platform.system().lower()
  233. my_machine = platform.machine().lower()
  234. if my_platform == 'linux':
  235. if 'ppc64le' in platform.platform():
  236. my_platform = 'linux-ppc64le'
  237. elif 'aarch64' in platform.platform():
  238. my_platform = 'linux-aarch64'
  239. else:
  240. my_platform = 'linux_musl'
  241. if my_platform == 'darwin' and my_machine == 'arm64':
  242. my_platform = 'darwin-arm64'
  243. def _platform_key(target_platform):
  244. """match by max prefix length, prefer shortest"""
  245. def _key_for_platform(platform):
  246. return len(os.path.commonprefix([target_platform, platform])), -len(platform)
  247. return _key_for_platform
  248. best_key = max(meta.keys(), key=_platform_key(my_platform))
  249. value = meta[best_key]
  250. ya_name = {'win32': 'ya-bin.exe', 'win32-clang-cl': 'ya-bin.exe'}.get(best_key, 'ya-bin') # XXX
  251. ya_dir = _get_dir(value['urls'], value['md5'], ya_name)
  252. # Popen `args` must have `str` type
  253. ya_path = str(os.path.join(ya_dir, ya_name))
  254. env = os.environ.copy()
  255. if 'YA_SOURCE_ROOT' not in env:
  256. src_root = _mine_repo_root()
  257. if src_root is not None:
  258. env['YA_SOURCE_ROOT'] = src_root
  259. # Disable respawn for opensource/ya
  260. if __file__.endswith('ya/opensource/ya'):
  261. env['YA_NO_RESPAWN'] = os.environ.get('YA_NO_RESPAWN', '1')
  262. for env_name in [
  263. 'LD_PRELOAD',
  264. 'Y_PYTHON_SOURCE_ROOT',
  265. ]:
  266. if env_name in os.environ:
  267. sys.stderr.write(
  268. "Warn: {}='{}' is specified and may affect the correct operation of the ya\n".format(
  269. env_name, env[env_name]
  270. )
  271. )
  272. if os.name == 'nt':
  273. import subprocess
  274. p = subprocess.Popen([ya_path] + result_args, env=env)
  275. p.wait()
  276. sys.exit(p.returncode)
  277. else:
  278. os.execve(ya_path, [ya_path] + result_args, env)
  279. if __name__ == '__main__':
  280. try:
  281. main()
  282. except Exception as e:
  283. sys.stderr.write('ERROR: ' + str(e) + '\n')
  284. from traceback import format_exc
  285. sys.stderr.write(format_exc() + "\n")
  286. sys.exit(1)