sync.py 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329
  1. from __future__ import annotations
  2. import importlib.metadata
  3. import os
  4. import shlex
  5. import subprocess
  6. from devenv import constants
  7. from devenv.lib import colima, config, fs, limactl, proc, venv
  8. # TODO: need to replace this with a nicer process executor in devenv.lib
  9. def run_procs(
  10. repo: str,
  11. reporoot: str,
  12. venv_path: str,
  13. _procs: tuple[tuple[str, tuple[str, ...], dict[str, str]], ...],
  14. verbose: bool = False,
  15. ) -> bool:
  16. procs: list[tuple[str, tuple[str, ...], subprocess.Popen[bytes]]] = []
  17. stdout = subprocess.PIPE if not verbose else None
  18. stderr = subprocess.STDOUT if not verbose else None
  19. for name, cmd, extra_env in _procs:
  20. print(f"⏳ {name}")
  21. if constants.DEBUG:
  22. proc.xtrace(cmd)
  23. env = {
  24. **constants.user_environ,
  25. **proc.base_env,
  26. "VIRTUAL_ENV": venv_path,
  27. "PATH": f"{venv_path}/bin:{reporoot}/.devenv/bin:{proc.base_path}",
  28. }
  29. if extra_env:
  30. env = {**env, **extra_env}
  31. procs.append(
  32. (
  33. name,
  34. cmd,
  35. subprocess.Popen(
  36. cmd,
  37. stdout=stdout,
  38. stderr=stderr,
  39. env=env,
  40. cwd=reporoot,
  41. ),
  42. )
  43. )
  44. all_good = True
  45. for name, final_cmd, p in procs:
  46. out, _ = p.communicate()
  47. if p.returncode != 0:
  48. all_good = False
  49. out_str = f"Output:\n{out.decode()}" if not verbose else ""
  50. print(
  51. f"""
  52. ❌ {name}
  53. failed command (code {p.returncode}):
  54. {shlex.join(final_cmd)}
  55. {out_str}
  56. """
  57. )
  58. else:
  59. print(f"✅ {name}")
  60. return all_good
  61. # Temporary, see https://github.com/getsentry/sentry/pull/78881
  62. def check_minimum_version(minimum_version: str):
  63. version = importlib.metadata.version("sentry-devenv")
  64. parsed_version = tuple(map(int, version.split(".")))
  65. parsed_minimum_version = tuple(map(int, minimum_version.split(".")))
  66. if parsed_version < parsed_minimum_version:
  67. raise SystemExit(
  68. f"""
  69. Hi! To reduce potential breakage we've defined a minimum
  70. devenv version ({minimum_version}) to run sync.
  71. Please run the following to update your global devenv:
  72. devenv update
  73. Then, use it to run sync this one time.
  74. {constants.root}/bin/devenv sync
  75. """
  76. )
  77. def main(context: dict[str, str]) -> int:
  78. check_minimum_version("1.14.2")
  79. repo = context["repo"]
  80. reporoot = context["reporoot"]
  81. repo_config = config.get_config(f"{reporoot}/devenv/config.ini")
  82. # TODO: context["verbose"]
  83. verbose = os.environ.get("SENTRY_DEVENV_VERBOSE") is not None
  84. FRONTEND_ONLY = os.environ.get("SENTRY_DEVENV_FRONTEND_ONLY") is not None
  85. USE_NEW_DEVSERVICES = os.environ.get("USE_NEW_DEVSERVICES") == "1"
  86. from devenv.lib import node
  87. node.install(
  88. repo_config["node"]["version"],
  89. repo_config["node"][constants.SYSTEM_MACHINE],
  90. repo_config["node"][f"{constants.SYSTEM_MACHINE}_sha256"],
  91. reporoot,
  92. )
  93. node.install_yarn(repo_config["node"]["yarn_version"], reporoot)
  94. # no more imports from devenv past this point! if the venv is recreated
  95. # then we won't have access to devenv libs until it gets reinstalled
  96. # venv's still needed for frontend because repo-local devenv and pre-commit
  97. # exist inside it
  98. venv_dir, python_version, requirements, editable_paths, bins = venv.get(reporoot, repo)
  99. url, sha256 = config.get_python(reporoot, python_version)
  100. print(f"ensuring {repo} venv at {venv_dir}...")
  101. venv.ensure(venv_dir, python_version, url, sha256)
  102. if constants.DARWIN and os.path.exists(f"{constants.root}/bin/colima"):
  103. # `devenv update`ing to >=1.14.0 will install global colima
  104. # so if it's there, uninstall the repo local stuff
  105. binroot = f"{reporoot}/.devenv/bin"
  106. colima.uninstall(binroot)
  107. limactl.uninstall(binroot)
  108. if not run_procs(
  109. repo,
  110. reporoot,
  111. venv_dir,
  112. (
  113. # TODO: devenv should provide a job runner (jobs run in parallel, tasks run sequentially)
  114. (
  115. "python dependencies (1/4)",
  116. (
  117. # upgrading pip first
  118. "pip",
  119. "install",
  120. "--constraint",
  121. "requirements-dev-frozen.txt",
  122. "pip",
  123. ),
  124. {},
  125. ),
  126. ),
  127. verbose,
  128. ):
  129. return 1
  130. if not run_procs(
  131. repo,
  132. reporoot,
  133. venv_dir,
  134. (
  135. (
  136. # Spreading out the network load by installing js,
  137. # then py in the next batch.
  138. "javascript dependencies (1/1)",
  139. (
  140. "yarn",
  141. "install",
  142. "--frozen-lockfile",
  143. "--no-progress",
  144. "--non-interactive",
  145. ),
  146. {
  147. "NODE_ENV": "development",
  148. },
  149. ),
  150. (
  151. "python dependencies (2/4)",
  152. (
  153. "pip",
  154. "uninstall",
  155. "-qqy",
  156. "djangorestframework-stubs",
  157. "django-stubs",
  158. ),
  159. {},
  160. ),
  161. ),
  162. verbose,
  163. ):
  164. return 1
  165. if not run_procs(
  166. repo,
  167. reporoot,
  168. venv_dir,
  169. (
  170. # could opt out of syncing python if FRONTEND_ONLY but only if repo-local devenv
  171. # and pre-commit were moved to inside devenv and not the sentry venv
  172. (
  173. "python dependencies (3/4)",
  174. (
  175. "pip",
  176. "install",
  177. "--constraint",
  178. "requirements-dev-frozen.txt",
  179. "-r",
  180. "requirements-dev-frozen.txt",
  181. ),
  182. {},
  183. ),
  184. ),
  185. verbose,
  186. ):
  187. return 1
  188. if not run_procs(
  189. repo,
  190. reporoot,
  191. venv_dir,
  192. (
  193. (
  194. "python dependencies (4/4)",
  195. ("python3", "-m", "tools.fast_editable", "--path", "."),
  196. {},
  197. ),
  198. ("pre-commit dependencies", ("pre-commit", "install", "--install-hooks", "-f"), {}),
  199. ),
  200. verbose,
  201. ):
  202. return 1
  203. fs.ensure_symlink("../../config/hooks/post-merge", f"{reporoot}/.git/hooks/post-merge")
  204. if not os.path.exists(f"{constants.home}/.sentry/config.yml") or not os.path.exists(
  205. f"{constants.home}/.sentry/sentry.conf.py"
  206. ):
  207. proc.run((f"{venv_dir}/bin/sentry", "init", "--dev"))
  208. # Frontend engineers don't necessarily always have devservices running and
  209. # can configure to skip them to save on local resources
  210. if FRONTEND_ONLY:
  211. print("Skipping python migrations since SENTRY_DEVENV_FRONTEND_ONLY is set.")
  212. return 0
  213. if USE_NEW_DEVSERVICES:
  214. # Ensure old sentry devservices is not being used, otherwise ports will conflict
  215. proc.run(
  216. (
  217. f"{venv_dir}/bin/{repo}",
  218. "devservices",
  219. "down",
  220. ),
  221. pathprepend=f"{reporoot}/.devenv/bin",
  222. exit=True,
  223. )
  224. proc.run(
  225. (f"{venv_dir}/bin/devservices", "up", "--mode", "migrations"),
  226. pathprepend=f"{reporoot}/.devenv/bin",
  227. exit=True,
  228. )
  229. else:
  230. # TODO: check healthchecks for redis and postgres to short circuit this
  231. proc.run(
  232. (
  233. f"{venv_dir}/bin/{repo}",
  234. "devservices",
  235. "up",
  236. "redis",
  237. "postgres",
  238. ),
  239. pathprepend=f"{reporoot}/.devenv/bin",
  240. exit=True,
  241. )
  242. if not run_procs(
  243. repo,
  244. reporoot,
  245. venv_dir,
  246. (
  247. (
  248. "python migrations",
  249. ("make", "apply-migrations"),
  250. {},
  251. ),
  252. ),
  253. verbose,
  254. ):
  255. return 1
  256. postgres_container = (
  257. "sentry_postgres" if os.environ.get("USE_NEW_DEVSERVICES") != "1" else "sentry-postgres-1"
  258. )
  259. # faster prerequisite check than starting up sentry and running createuser idempotently
  260. stdout = proc.run(
  261. (
  262. "docker",
  263. "exec",
  264. postgres_container,
  265. "psql",
  266. "sentry",
  267. "postgres",
  268. "-t",
  269. "-c",
  270. "select exists (select from auth_user where email = 'admin@sentry.io')",
  271. ),
  272. stdout=True,
  273. )
  274. if stdout != "t":
  275. proc.run(
  276. (
  277. f"{venv_dir}/bin/sentry",
  278. "createuser",
  279. "--superuser",
  280. "--email",
  281. "admin@sentry.io",
  282. "--password",
  283. "admin",
  284. "--no-input",
  285. )
  286. )
  287. return 0