sentry/bin/split-silo-database

#!/usr/bin/env python

from typing import List

import click
from django.apps import apps

import docker
from sentry.runner import configure
from sentry.silo.base import SiloMode

configure()


def exec_run(container, command):
    wrapped_command = f'sh -c "{" ".join(command)}"'
    exit_code, output = container.exec_run(cmd=wrapped_command, stdout=True, stderr=True)
    if exit_code:
        click.echo("Container operation Failed!")
        click.echo(f"Container operation failed with {output}")


def split_database(tables: List[str], source: str, destination: str, reset: bool, verbose: bool):
    click.echo(f">> Dumping tables from {source} database")
    command = ["pg_dump", "-U", "postgres", "-d", source, "--clean"]
    for table in tables:
        command.extend(["-t", table])
    command.extend([">", f"/tmp/{destination}-tables.sql"])

    client = docker.from_env()
    postgres = client.containers.get("sentry_postgres")

    if verbose:
        click.echo(f">> Running {' '.join(command)}")
    exec_run(postgres, command)

    if reset:
        click.echo(f">> Dropping existing {destination} database")
        exec_run(postgres, ["dropdb", "-U", "postgres", "--if-exists", destination])
        exec_run(postgres, ["createdb", "-U", "postgres", destination])

    # Use the dump file to build control silo tables.
    click.echo(f">> Building {destination} database from dump file")
    import_command = ["psql", "-U", "postgres", destination, "<", f"/tmp/{destination}-tables.sql"]
    if verbose:
        click.echo(f">> Running {' '.join(import_command)}")
    exec_run(postgres, import_command)


@click.command()
@click.option("--verbose", default=False, is_flag=True, help="Enable verbose logging")
@click.option(
    "--reset",
    default=False,
    is_flag=True,
    help="Reset the target databases to be empty before loading extracted data and schema.",
)
@click.option("--database", default="sentry", help="Which database to derive splits from")
def main(database: str, reset: bool, verbose: bool):
    """
    This is a development tool that can convert a monolith database into
    control + region databases by using silo annotations.

    This operation will not modify the original source database.
    """
    region_tables = ["django_migrations"]
    control_tables = ["django_migrations"]
    for model in apps.get_models():
        silo_limit = getattr(model._meta, "silo_limit", None)
        if not silo_limit:
            click.echo(f"> Could not find silo assignment for {model._meta.db_table}")
            continue
        if SiloMode.CONTROL in silo_limit.modes:
            control_tables.append(model._meta.db_table)
        if SiloMode.REGION in silo_limit.modes:
            region_tables.append(model._meta.db_table)

    split_database(control_tables, database, "control", reset=reset, verbose=verbose)
    split_database(region_tables, database, "region", reset=reset, verbose=verbose)


if __name__ == "__main__":
    main()