dev: move core make macros into devenv sync (#68379)

Makefile

@@ -3,24 +3,28 @@ all: develop
 
 PIP := python -m pip --disable-pip-version-check
 WEBPACK := yarn build-acceptance
-POSTGRES_CONTAINER := sentry_postgres
 
 freeze-requirements:
 	@python3 -S -m tools.freeze_requirements
 
 bootstrap \
 develop \
+install-js-dev \
+install-py-dev \
+apply-migrations: devenv-sync ;
+
+# This is to ensure devenv sync's only called once if the above
+# aliases are combined e.g. `make install-js-dev install-py-dev`
+.PHONY: devenv-sync
+devenv-sync:
+	devenv sync
+
 clean \
-init-config \
-run-dependent-services \
 drop-db \
 create-db \
-apply-migrations \
 reset-db \
 setup-git \
-node-version-check \
-install-js-dev \
-install-py-dev :
+node-version-check :
 	@./scripts/do.sh $@
 
 build-platform-assets \

devenv/sync.py

@@ -69,6 +69,9 @@ def main(context: dict[str, str]) -> int:
     repo = context["repo"]
     reporoot = context["reporoot"]
 
+    # don't want this to be accidentally run from getsentry
+    assert repo == "sentry", repo
+
     venv_dir, python_version, requirements, editable_paths, bins = venv.get(reporoot, repo)
     url, sha256 = config.get_python(reporoot, python_version)
     print(f"ensuring {repo} venv at {venv_dir}...")
@@ -112,8 +115,39 @@ def main(context: dict[str, str]) -> int:
         reporoot,
         venv_dir,
         (
-            ("javascript dependencies", ("make", "install-js-dev")),
-            ("python dependencies", ("make", "install-py-dev")),
+            (
+                "javascript dependencies",
+                (
+                    "bash",
+                    "-euo",
+                    "pipefail",
+                    "-c",
+                    """
+NODE_ENV=development ./.devenv/bin/yarn install --frozen-lockfile
+yarn check --verify-tree || yarn install --check-files
+""",
+                ),
+            ),
+            (
+                "python dependencies",
+                (
+                    "bash",
+                    "-euo",
+                    "pipefail",
+                    "-c",
+                    """
+# install pinned pip
+pip install --constraint requirements-dev-frozen.txt pip
+
+# pip doesn't do well with swapping drop-ins
+pip uninstall -qqy djangorestframework-stubs django-stubs
+
+pip install -r requirements-dev-frozen.txt
+
+python3 -m tools.fast_editable --path .
+""",
+                ),
+            ),
         ),
     ):
         return 1
@@ -136,12 +170,12 @@ def main(context: dict[str, str]) -> int:
     if not os.path.exists(f"{constants.home}/.sentry/config.yml") or not os.path.exists(
         f"{constants.home}/.sentry/sentry.conf.py"
     ):
-        proc.run((f"{venv_dir}/bin/sentry", "init", "--dev"))
+        proc.run((f"{venv_dir}/bin/sentry", "init", "--dev", "--no-clobber"))
 
     # TODO: check healthchecks for redis and postgres to short circuit this
     proc.run(
         (
-            f"{venv_dir}/bin/{repo}",
+            f"{venv_dir}/bin/sentry",
             "devservices",
             "up",
             "redis",
@@ -151,17 +185,55 @@ def main(context: dict[str, str]) -> int:
         exit=True,
     )
 
-    if run_procs(
+    if not run_procs(
         repo,
         reporoot,
         venv_dir,
         (
             (
                 "python migrations",
-                ("make", "apply-migrations"),
+                (
+                    "bash",
+                    "-euo",
+                    "pipefail",
+                    "-c",
+                    """
+make create-db
+sentry upgrade --noinput
+""",
+                ),
             ),
         ),
     ):
-        return 0
+        return 1
+
+    # Starting sentry is slow.
+    stdout = proc.run(
+        (
+            "docker",
+            "exec",
+            "sentry_postgres",
+            "psql",
+            "sentry",
+            "postgres",
+            "-t",
+            "-c",
+            "select exists (select from auth_user where email = 'admin@sentry.io')",
+        ),
+        stdout=True,
+    )
+    if stdout != "t":
+        proc.run(
+            (
+                f"{venv_dir}/bin/sentry",
+                "createuser",
+                "--superuser",
+                "--email",
+                "admin@sentry.io",
+                "--password",
+                "admin",
+                "--no-input",
+            )
+        )
 
-    return 1
+    return 0

scripts/ensure-venv.sh

@@ -24,7 +24,7 @@ if [[ -n "$VIRTUAL_ENV" ]]; then
     query-valid-python-version || exit 1
 else
     if [[ ! -f "${venv_name}/bin/activate" ]]; then
-        die "You don't seem to have a virtualenv. Please create one by running: source ./scripts/bootstrap-py3-venv"
+        die "You don't seem to have a virtualenv. Please create one by running: devenv sync"
     fi
     die "You have a virtualenv, but it doesn't seem to be activated. Please run: source ${venv_name}/bin/activate"
 fi

scripts/lib.sh

@@ -78,33 +78,16 @@ sudo-askpass() {
     fi
 }
 
+# needed by getsentry make
 pip-install() {
     pip install --constraint "${HERE}/../requirements-dev-frozen.txt" "$@"
 }
 
+# needed by getsentry make
 upgrade-pip() {
     pip-install pip
 }
 
-install-py-dev() {
-    upgrade-pip
-    # It places us within top src dir to be at the same path as setup.py
-    # This helps when getsentry calls into this script
-    cd "${HERE}/.." || exit
-
-    echo "--> Installing Sentry (for development)"
-
-    # pip doesn't do well with swapping drop-ins
-    pip uninstall -qqy djangorestframework-stubs django-stubs
-
-    pip-install -r requirements-dev-frozen.txt
-
-    # SENTRY_LIGHT_BUILD=1 disables webpacking during setup.py.
-    # Webpacked assets are only necessary for devserver (which does it lazily anyways)
-    # and acceptance tests, which webpack automatically if run.
-    python3 -m tools.fast_editable --path .
-}
-
 setup-git-config() {
     git config --local branch.autosetuprebase always
     git config --local core.ignorecase false
@@ -138,32 +121,6 @@ node-version-check() {
         )
 }
 
-install-js-dev() {
-    node-version-check
-    echo "--> Installing Yarn packages (for development)"
-    # Use NODE_ENV=development so that yarn installs both dependencies + devDependencies
-    NODE_ENV=development yarn install --frozen-lockfile
-    # A common problem is with node packages not existing in `node_modules` even though `yarn install`
-    # says everything is up to date. Even though `yarn install` is run already, it doesn't take into
-    # account the state of the current filesystem (it only checks .yarn-integrity).
-    # Add an additional check against `node_modules`
-    yarn check --verify-tree || yarn install --check-files
-}
-
-develop() {
-    install-js-dev
-    install-py-dev
-    setup-git
-}
-
-init-config() {
-    sentry init --dev --no-clobber
-}
-
-run-dependent-services() {
-    sentry devservices up
-}
-
 create-db() {
     container_name=${POSTGRES_CONTAINER:-sentry_postgres}
     echo "--> Creating 'sentry' database"
@@ -173,22 +130,6 @@ create-db() {
     docker exec "${container_name}" createdb -h 127.0.0.1 -U postgres -E utf-8 region || true
 }
 
-apply-migrations() {
-    create-db
-    echo "--> Applying migrations"
-    sentry upgrade --noinput
-}
-
-create-superuser() {
-    echo "--> Creating a superuser account"
-    if [[ -n "${GITHUB_ACTIONS+x}" ]]; then
-        sentry createuser --superuser --email foo@tbd.com --no-password --no-input
-    else
-        sentry createuser --superuser --email admin@sentry.io --password admin --no-input
-        echo "Password is admin."
-    fi
-}
-
 build-platform-assets() {
     echo "--> Building platform assets"
     echo "from sentry.utils.integrationdocs import sync_docs; sync_docs(quiet=True)" | sentry exec
@@ -196,18 +137,6 @@ build-platform-assets() {
     test -f src/sentry/integration-docs/android.json
 }
 
-bootstrap() {
-    develop
-    init-config
-    run-dependent-services
-    apply-migrations
-    create-superuser
-    # Load mocks requires a superuser
-    bin/load-mocks
-    build-platform-assets
-    echo "--> Finished bootstrapping. Have a nice day."
-}
-
 clean() {
     echo "--> Cleaning static cache"
     rm -rf dist/* src/sentry/static/sentry/dist/*
@@ -231,8 +160,7 @@ drop-db() {
 
 reset-db() {
     drop-db
-    apply-migrations
-    create-superuser
+    devenv sync
     echo 'Finished resetting database. To load mock data, run `./bin/load-mocks`'
 }