sentry/bin/load-mocks

#!/usr/bin/env python
from sentry.runner import configure
configure()

import itertools
import six

from datetime import datetime, timedelta
from django.conf import settings
from django.db import IntegrityError, transaction
from django.utils import timezone
from hashlib import sha1
from pytz import utc
from random import randint
from uuid import uuid4

from sentry import roles
from sentry.app import tsdb, buffer
from sentry.models import (
    Activity, Broadcast, File, GroupMeta, Organization, OrganizationAccessRequest,
    OrganizationMember, Project, Release, ReleaseFile, Team, User, UserReport,
    GroupRelease, Environment, ReleaseEnvironment, ReleaseCommit, Commit, Repository,
    CommitAuthor, CommitFileChange,
)
from sentry.signals import mocks_loaded
from sentry.utils.hashlib import md5_text
from sentry.utils.samples import create_sample_event

PLATFORMS = itertools.cycle([
    'ruby',
    'php',
    'python',
    'java',
    'javascript',
])

LEVELS = itertools.cycle([
    'error',
    'error',
    'error',
    'fatal',
    'warning',
])

ENVIRONMENTS = itertools.cycle([
    'production',
    'production',
    'staging',
    'alpha',
    'beta',
])


def create_system_time_series():
    now = datetime.utcnow().replace(tzinfo=utc)

    for _ in xrange(60):
        count = randint(1, 10)
        tsdb.incr_multi((
            (tsdb.models.internal, 'client-api.all-versions.responses.2xx'),
            (tsdb.models.internal, 'client-api.all-versions.requests'),
        ), now, int(count * 0.9))
        tsdb.incr_multi((
            (tsdb.models.internal, 'client-api.all-versions.responses.4xx'),
        ), now, int(count * 0.05))
        tsdb.incr_multi((
            (tsdb.models.internal, 'client-api.all-versions.responses.5xx'),
        ), now, int(count * 0.1))
        now = now - timedelta(seconds=1)

    for _ in xrange(24 * 30):
        count = randint(100, 1000)
        tsdb.incr_multi((
            (tsdb.models.internal, 'client-api.all-versions.responses.2xx'),
            (tsdb.models.internal, 'client-api.all-versions.requests'),
        ), now, int(count * 4.9))
        tsdb.incr_multi((
            (tsdb.models.internal, 'client-api.all-versions.responses.4xx'),
        ), now, int(count * 0.05))
        tsdb.incr_multi((
            (tsdb.models.internal, 'client-api.all-versions.responses.5xx'),
        ), now, int(count * 0.1))
        now = now - timedelta(hours=1)


def create_sample_time_series(event, release=None):
    group = event.group
    project = group.project

    now = datetime.utcnow().replace(tzinfo=utc)

    environment = Environment.get_or_create(
        project=project,
        name=event.get_tag('environment') or ''
    )

    if release:
        ReleaseEnvironment.get_or_create(
            project=project,
            release=release,
            environment=environment,
            datetime=now,
        )

        grouprelease = GroupRelease.get_or_create(
            group=group,
            release=release,
            environment=environment,
            datetime=now,
        )

    for _ in xrange(60):
        count = randint(1, 10)
        tsdb.incr_multi((
            (tsdb.models.project, project.id),
            (tsdb.models.group, group.id),
        ), now, count)
        tsdb.incr_multi((
            (tsdb.models.organization_total_received, project.organization_id),
            (tsdb.models.project_total_received, project.id),
        ), now, int(count * 1.1))
        tsdb.incr_multi((
            (tsdb.models.organization_total_rejected, project.organization_id),
            (tsdb.models.project_total_rejected, project.id),
        ), now, int(count * 0.1))

        frequencies = [
            (tsdb.models.frequent_projects_by_organization, {
                project.organization_id: {
                    project.id: count,
                },
            }),
            (tsdb.models.frequent_issues_by_project, {
                project.id: {
                    group.id: count,
                },
            }),
            (tsdb.models.frequent_environments_by_group, {
                group.id: {
                    environment.id: count,
                },
            })
        ]
        if release:
            frequencies.append(
                (tsdb.models.frequent_releases_by_group, {
                    group.id: {
                        grouprelease.id: count,
                    },
                })
            )

        tsdb.record_frequency_multi(frequencies, now)

        now = now - timedelta(seconds=1)

    for _ in xrange(24 * 30):
        count = randint(100, 1000)
        tsdb.incr_multi((
            (tsdb.models.project, group.project.id),
            (tsdb.models.group, group.id),
        ), now, count)
        tsdb.incr_multi((
            (tsdb.models.organization_total_received, group.project.organization_id),
            (tsdb.models.project_total_received, group.project.id),
        ), now, int(count * 1.1))
        tsdb.incr_multi((
            (tsdb.models.organization_total_rejected, group.project.organization_id),
            (tsdb.models.project_total_rejected, group.project.id),
        ), now, int(count * 0.1))

        frequencies = [
            (tsdb.models.frequent_projects_by_organization, {
                project.organization_id: {
                    project.id: count,
                },
            }),
            (tsdb.models.frequent_issues_by_project, {
                project.id: {
                    group.id: count,
                },
            }),
            (tsdb.models.frequent_environments_by_group, {
                group.id: {
                    environment.id: count,
                },
            })
        ]
        if release:
            frequencies.append(
                (tsdb.models.frequent_releases_by_group, {
                    group.id: {
                        grouprelease.id: count,
                    },
                })
            )

        tsdb.record_frequency_multi(frequencies, now)

        now = now - timedelta(hours=1)


def main(num_events=1):
    user = User.objects.filter(is_superuser=True)[0]

    dummy_user, _ = User.objects.get_or_create(
        username='dummy@example.com',
        defaults={
            'email': 'dummy@example.com',
        }
    )
    dummy_user.set_password('dummy')
    dummy_user.save()

    mocks = (
        ('Massive Dynamic', ('Ludic Science',)),
        ('Captain Planet', ('Earth', 'Fire', 'Wind', 'Water', 'Heart')),
    )

    Broadcast.objects.create(
        title="Learn about Source Maps",
        message="Source maps are JSON files that contain information on how to map your transpiled source code back to their original source.",
        link="https://docs.sentry.io/hosted/clients/javascript/sourcemaps/#uploading-source-maps-to-sentry",
    )

    if settings.SENTRY_SINGLE_ORGANIZATION:
        org = Organization.get_default()
    else:
        print('Mocking org {}'.format('Default'))
        org, _ = Organization.objects.get_or_create(
            slug='default',
        )

    OrganizationMember.objects.get_or_create(
        user=user,
        organization=org,
        role=roles.get_top_dog().id,
    )

    dummy_member, _ = OrganizationMember.objects.get_or_create(
        user=dummy_user,
        organization=org,
        defaults={
            'role': roles.get_default().id,
        }
    )

    for team_name, project_names in mocks:
        print('> Mocking team {}'.format(team_name))
        team, _ = Team.objects.get_or_create(
            name=team_name,
            defaults={
                'organization': org,
            },
        )

        for project_name in project_names:
            print('  > Mocking project {}'.format(project_name))
            project, _ = Project.objects.get_or_create(
                team=team,
                name=project_name,
                defaults={
                    'organization': org,
                    'first_event': timezone.now(),
                }
            )
            if not project.first_event:
                project.update(
                    first_event=project.date_added,
                )

            with transaction.atomic():
                has_release = Release.objects.filter(
                    version=sha1(uuid4().bytes).hexdigest(),
                    organization_id=project.organization_id,
                    projects=project
                ).exists()
                if not has_release:
                    release = Release.objects.filter(
                        version=sha1(uuid4().bytes).hexdigest(),
                        organization_id=project.organization_id,
                    ).first()
                    if not release:
                        release = Release.objects.create(
                            version=sha1(uuid4().bytes).hexdigest(),
                            organization_id=project.organization_id
                        )
                    release.add_project(project)

            ReleaseFile.objects.get_or_create(
                organization_id=project.organization_id,
                release=release,
                name='an-example.js',
                file=File.objects.get_or_create(
                    name='an-example.js',
                    type='release.file',
                    checksum='abcde' * 8,
                    size=13043,
                )[0],
                defaults={'organization_id': project.organization_id}
            )

            raw_commits = [
                {
                    'key': '6fb7672bee9c469a8aa074f52ed72eddc939ae7d',
                    'message': 'Fix widget for real this time\n\nTurns out the sprocket configuration was broken.',
                    'author': (user.name, user.email),
                    'files': [
                        ('/static/js/widget.js', 'M'),
                    ],
                },
                {
                    'key': '98a0672968c19e3d7921def5d724cc977e85f527',
                    'message': 'Added .gitignore',
                    'author': ('Jane Doe', 'janedoe@example.com'),
                    'files': [
                        ('.gitignore', 'A'),
                        ('.DS_Store', 'D')
                    ],
                },
                {
                    'key': '631cd9096bd9811a046a472bb0aa8b573e86e1f1',
                    'message': 'Fix widget',
                    'author': (user.name, user.email),
                    'files': [
                        ('/static/html/widget.html', 'M'),
                        ('/static/js/widget.js', 'M'),
                        ('.DS_Store', 'A'),
                    ],
                },
            ]

            for commit_index, raw_commit in enumerate(raw_commits):
                commit = Commit.objects.get_or_create(
                    organization_id=org.id,
                    repository_id=Repository.objects.get_or_create(
                        organization_id=org.id,
                        name='Example Repo',
                        provider='github',
                        external_id='example/example',
                    )[0].id,
                    key=raw_commit['key'],
                    author=CommitAuthor.objects.get_or_create(
                        organization_id=org.id,
                        name=raw_commit['author'][0],
                        email=raw_commit['author'][1],
                    )[0],
                    message=raw_commit['message'],
                )[0]

                for file in raw_commit['files']:
                    CommitFileChange.objects.get_or_create(
                        organization_id=org.id,
                        commit=commit,
                        filename=file[0],
                        type=file[1],
                    )

                ReleaseCommit.objects.get_or_create(
                    organization_id=org.id,
                    release=release,
                    commit=commit,
                    order=commit_index,
                )

            Activity.objects.create(
                type=Activity.RELEASE,
                project=project,
                ident=release.version,
                user=user,
                data={'version': release.version},
            )

            # Add a bunch of additional dummy events to support pagination
            last_event = None
            for _ in range(45):
                platform = PLATFORMS.next()

                last_event = create_sample_event(
                    project=project,
                    platform=platform,
                    release=release.version,
                    level=LEVELS.next(),
                    environment=ENVIRONMENTS.next(),
                    message='This is a mostly useless example %s exception' % platform,
                    checksum=md5_text(platform + six.text_type(_)).hexdigest(),
                )

            for _ in range(num_events):
                event1 = create_sample_event(
                    project=project,
                    platform='python',
                    release=release.version,
                    environment=ENVIRONMENTS.next(),
                )

                event2 = create_sample_event(
                    project=project,
                    platform='javascript',
                    release=release.version,
                    environment=ENVIRONMENTS.next(),
                    sdk={
                        'name': 'raven-js',
                        'version': '2.1.0',
                    },
                )

                event3 = create_sample_event(project, 'java')

                event4 = create_sample_event(
                    project=project,
                    platform='ruby',
                    release=release.version,
                    environment=ENVIRONMENTS.next(),
                )

                event5 = create_sample_event(
                    project=project,
                    platform='cocoa',
                    release=release.version,
                    environment=ENVIRONMENTS.next(),
                )

                create_sample_event(
                    project=project,
                    platform='php',
                    release=release.version,
                    environment=ENVIRONMENTS.next(),
                    message='This is a an example PHP event with an extremely long and annoying title\nIt also happens to contain some newlines in it,\nthus making it even more annoying.',
                )

            create_sample_event(
                project=project,
                environment=ENVIRONMENTS.next(),
                platform='csp',
            )

            with transaction.atomic():
                try:
                    GroupMeta.objects.create(
                        group=event1.group,
                        key='github:tid',
                        value='134',
                    )
                except IntegrityError:
                    pass

            UserReport.objects.create(
                project=project,
                event_id=event3.event_id,
                group=event3.group,
                name='Jane Doe',
                email='jane@example.com',
                comments='I have no idea how I got here.',
            )

            print('    > Loading time series data'.format(project_name))

            create_sample_time_series(event1, release=release)
            create_sample_time_series(event2, release=release)
            create_sample_time_series(event3)
            create_sample_time_series(event4, release=release)
            create_sample_time_series(event5, release=release)

            if hasattr(buffer, 'process_pending'):
                print('    > Processing pending buffers')
                buffer.process_pending()

            mocks_loaded.send(project=project, sender=__name__)

        OrganizationAccessRequest.objects.create_or_update(
            member=dummy_member,
            team=team,
        )

    Activity.objects.create(
        type=Activity.RELEASE,
        project=project,
        ident='4f38b65c62c4565aa94bba391ff8946922a8eed4',
        user=user,
        data={'version': '4f38b65c62c4565aa94bba391ff8946922a8eed4'},
    )

    create_system_time_series()


if __name__ == '__main__':
    settings.CELERY_ALWAYS_EAGER = True

    from optparse import OptionParser

    parser = OptionParser()
    parser.add_option('--events', dest='num_events', default=1, type=int)

    (options, args) = parser.parse_args()

    try:
        main(num_events=options.num_events)
    except Exception:
        # Avoid reporting any issues recursively back into Sentry
        import traceback
        import sys

        traceback.print_exc()
        sys.exit(1)