sentry/tests/snuba/api/endpoints/test_organization_stats_v2.py

from datetime import datetime, timedelta, timezone

from sentry.constants import DataCategory
from sentry.testutils.cases import APITestCase, OutcomesSnubaTest
from sentry.testutils.helpers.datetime import freeze_time
from sentry.testutils.silo import region_silo_test
from sentry.utils.outcomes import Outcome


@region_silo_test
class OrganizationStatsTestV2(APITestCase, OutcomesSnubaTest):
    endpoint = "sentry-api-0-organization-stats-v2"

    def setUp(self):
        super().setUp()
        self.now = datetime(2021, 3, 14, 12, 27, 28, tzinfo=timezone.utc)

        self.login_as(user=self.user)

        self.org = self.organization
        self.org.flags.allow_joinleave = False
        self.org.save()

        self.org2 = self.create_organization()
        self.org3 = self.create_organization()

        self.project = self.create_project(
            name="bar", teams=[self.create_team(organization=self.org, members=[self.user])]
        )
        self.project2 = self.create_project(
            name="foo", teams=[self.create_team(organization=self.org, members=[self.user])]
        )
        self.project3 = self.create_project(organization=self.org2)

        self.user2 = self.create_user(is_superuser=False)
        self.create_member(user=self.user2, organization=self.organization, role="member", teams=[])
        self.create_member(user=self.user2, organization=self.org3, role="member", teams=[])
        self.project4 = self.create_project(
            name="users2sproj",
            teams=[self.create_team(organization=self.org, members=[self.user2])],
        )

        self.store_outcomes(
            {
                "org_id": self.org.id,
                "timestamp": self.now - timedelta(hours=1),
                "project_id": self.project.id,
                "outcome": Outcome.ACCEPTED,
                "reason": "none",
                "category": DataCategory.ERROR,
                "quantity": 1,
            },
            5,
        )
        self.store_outcomes(
            {
                "org_id": self.org.id,
                "timestamp": self.now - timedelta(hours=1),
                "project_id": self.project.id,
                "outcome": Outcome.ACCEPTED,
                "reason": "none",
                "category": DataCategory.DEFAULT,  # test that this shows up under error
                "quantity": 1,
            }
        )
        self.store_outcomes(
            {
                "org_id": self.org.id,
                "timestamp": self.now - timedelta(hours=1),
                "project_id": self.project.id,
                "outcome": Outcome.RATE_LIMITED,
                "reason": "smart_rate_limit",
                "category": DataCategory.ATTACHMENT,
                "quantity": 1024,
            }
        )
        self.store_outcomes(
            {
                "org_id": self.org.id,
                "timestamp": self.now - timedelta(hours=1),
                "project_id": self.project2.id,
                "outcome": Outcome.RATE_LIMITED,
                "reason": "smart_rate_limit",
                "category": DataCategory.TRANSACTION,
                "quantity": 1,
            }
        )

    def do_request(self, query, user=None, org=None, status_code=200):
        self.login_as(user=user or self.user)
        org_slug = (org or self.organization).slug
        if status_code >= 400:
            return self.get_error_response(org_slug, **query, status_code=status_code)
        return self.get_success_response(org_slug, **query, status_code=status_code)

    def test_empty_request(self):
        response = self.do_request({}, status_code=400)
        assert result_sorted(response.data) == {"detail": 'At least one "field" is required.'}

    def test_inaccessible_project(self):
        response = self.do_request({"project": [self.project3.id]}, status_code=403)

        assert result_sorted(response.data) == {
            "detail": "You do not have permission to perform this action."
        }

    def test_no_projects_available(self):
        response = self.do_request(
            {
                "groupBy": ["project"],
                "statsPeriod": "1d",
                "interval": "1d",
                "field": ["sum(quantity)"],
                "category": ["error", "transaction"],
            },
            user=self.user2,
            org=self.org3,
            status_code=400,
        )

        assert result_sorted(response.data) == {
            "detail": "No projects available",
        }

    def test_unknown_field(self):
        response = self.do_request(
            {
                "field": ["summ(qarntenty)"],
                "statsPeriod": "1d",
                "interval": "1d",
            },
            status_code=400,
        )

        assert result_sorted(response.data) == {
            "detail": 'Invalid field: "summ(qarntenty)"',
        }

    def test_no_end_param(self):
        response = self.do_request(
            {"field": ["sum(quantity)"], "interval": "1d", "start": "2021-03-14T00:00:00Z"},
            status_code=400,
        )

        assert result_sorted(response.data) == {"detail": "start and end are both required"}

    @freeze_time(datetime(2021, 3, 14, 12, 27, 28, tzinfo=timezone.utc))
    def test_future_request(self):
        response = self.do_request(
            {
                "field": ["sum(quantity)"],
                "interval": "1h",
                "category": ["error"],
                "start": "2021-03-14T15:30:00",
                "end": "2021-03-14T16:30:00",
            },
            status_code=200,
        )

        assert result_sorted(response.data) == {
            "intervals": [
                "2021-03-14T12:00:00Z",
                "2021-03-14T13:00:00Z",
                "2021-03-14T14:00:00Z",
                "2021-03-14T15:00:00Z",
                "2021-03-14T16:00:00Z",
            ],
            "groups": [
                {
                    "by": {},
                    "series": {"sum(quantity)": [0, 0, 0, 0, 0]},
                    "totals": {"sum(quantity)": 0},
                }
            ],
            "start": "2021-03-14T12:00:00Z",
            "end": "2021-03-14T17:00:00Z",
        }

    def test_unknown_category(self):
        response = self.do_request(
            {
                "field": ["sum(quantity)"],
                "statsPeriod": "1d",
                "interval": "1d",
                "category": "scoobydoo",
            },
            status_code=400,
        )

        assert result_sorted(response.data) == {
            "detail": 'Invalid category: "scoobydoo"',
        }

    def test_unknown_outcome(self):
        response = self.do_request(
            {
                "field": ["sum(quantity)"],
                "statsPeriod": "1d",
                "interval": "1d",
                "category": "error",
                "outcome": "scoobydoo",
            },
            status_code=400,
        )

        assert result_sorted(response.data) == {
            "detail": 'Invalid outcome: "scoobydoo"',
        }

    def test_unknown_groupby(self):
        response = self.do_request(
            {
                "field": ["sum(quantity)"],
                "groupBy": ["category_"],
                "statsPeriod": "1d",
                "interval": "1d",
            },
            status_code=400,
        )

        assert result_sorted(response.data) == {"detail": 'Invalid groupBy: "category_"'}

    def test_resolution_invalid(self):
        self.do_request(
            {
                "statsPeriod": "1d",
                "interval": "bad_interval",
            },
            org=self.org,
            status_code=400,
        )

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_attachment_filter_only(self):
        response = self.do_request(
            {
                "project": [-1],
                "statsPeriod": "1d",
                "interval": "1d",
                "field": ["sum(quantity)"],
                "category": ["error", "attachment"],
            },
            status_code=400,
        )

        assert result_sorted(response.data) == {
            "detail": "if filtering by attachment no other category may be present"
        }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_timeseries_interval(self):
        response = self.do_request(
            {
                "project": [-1],
                "category": ["error"],
                "statsPeriod": "1d",
                "interval": "1d",
                "field": ["sum(quantity)"],
            },
            status_code=200,
        )

        assert result_sorted(response.data) == {
            "intervals": ["2021-03-13T00:00:00Z", "2021-03-14T00:00:00Z"],
            "groups": [
                {"by": {}, "series": {"sum(quantity)": [0, 6]}, "totals": {"sum(quantity)": 6}}
            ],
            "start": "2021-03-13T00:00:00Z",
            "end": "2021-03-15T00:00:00Z",
        }

        response = self.do_request(
            {
                "project": [-1],
                "statsPeriod": "1d",
                "interval": "6h",
                "field": ["sum(quantity)"],
                "category": ["error"],
            },
            status_code=200,
        )

        assert result_sorted(response.data) == {
            "intervals": [
                "2021-03-13T12:00:00Z",
                "2021-03-13T18:00:00Z",
                "2021-03-14T00:00:00Z",
                "2021-03-14T06:00:00Z",
                "2021-03-14T12:00:00Z",
            ],
            "groups": [
                {
                    "by": {},
                    "series": {"sum(quantity)": [0, 0, 0, 6, 0]},
                    "totals": {"sum(quantity)": 6},
                }
            ],
            "start": "2021-03-13T12:00:00Z",
            "end": "2021-03-14T18:00:00Z",
        }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_user_org_total_all_accessible(self):
        response = self.do_request(
            {
                "project": [-1],
                "statsPeriod": "1d",
                "interval": "1d",
                "field": ["sum(quantity)"],
                "category": ["error", "transaction"],
            },
            user=self.user2,
            status_code=200,
        )

        assert result_sorted(response.data) == {
            "start": "2021-03-13T00:00:00Z",
            "end": "2021-03-15T00:00:00Z",
            "intervals": ["2021-03-13T00:00:00Z", "2021-03-14T00:00:00Z"],
            "groups": [
                {"by": {}, "series": {"sum(quantity)": [0, 7]}, "totals": {"sum(quantity)": 7}}
            ],
        }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_user_no_proj_specific_access(self):
        response = self.do_request(
            {
                "project": self.project.id,
                "statsPeriod": "1d",
                "interval": "1d",
                "field": ["sum(quantity)"],
                "category": ["error", "transaction"],
            },
            user=self.user2,
            status_code=403,
        )

        response = self.do_request(
            {
                "project": [-1],
                "statsPeriod": "1d",
                "interval": "1d",
                "field": ["sum(quantity)"],
                "category": ["error", "transaction"],
                "groupBy": ["project"],
            },
            user=self.user2,
            status_code=200,
        )

        assert result_sorted(response.data) == {
            "start": "2021-03-13T00:00:00Z",
            "end": "2021-03-15T00:00:00Z",
            "groups": [],
        }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_no_project_access(self):
        user = self.create_user(is_superuser=False)
        self.create_member(user=user, organization=self.organization, role="member", teams=[])

        response = self.do_request(
            {
                "project": [self.project.id],
                "statsPeriod": "1d",
                "interval": "1d",
                "category": ["error", "transaction"],
                "field": ["sum(quantity)"],
            },
            org=self.organization,
            user=user,
            status_code=403,
        )

        assert result_sorted(response.data) == {
            "detail": "You do not have permission to perform this action."
        }

        response = self.do_request(
            {
                "project": [self.project.id],
                "groupBy": ["project"],
                "statsPeriod": "1d",
                "interval": "1d",
                "category": ["error", "transaction"],
                "field": ["sum(quantity)"],
            },
            org=self.organization,
            user=user,
            status_code=403,
        )

        assert result_sorted(response.data) == {
            "detail": "You do not have permission to perform this action."
        }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_open_membership_semantics(self):
        self.org.flags.allow_joinleave = True
        self.org.save()
        response = self.do_request(
            {
                "project": [-1],
                "statsPeriod": "1d",
                "interval": "1d",
                "field": ["sum(quantity)"],
                "category": ["error", "transaction"],
                "groupBy": ["project"],
            },
            user=self.user2,
            status_code=200,
        )

        assert result_sorted(response.data) == {
            "start": "2021-03-13T00:00:00Z",
            "end": "2021-03-15T00:00:00Z",
            "groups": [
                {
                    "by": {"project": self.project.id},
                    "totals": {"sum(quantity)": 6},
                },
                {
                    "by": {"project": self.project2.id},
                    "totals": {"sum(quantity)": 1},
                },
            ],
        }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_org_simple(self):
        response = self.do_request(
            {
                "statsPeriod": "2d",
                "interval": "1d",
                "field": ["sum(quantity)"],
                "groupBy": ["category", "outcome", "reason"],
            },
            org=self.org,
            status_code=200,
        )

        assert result_sorted(response.data) == {
            "start": "2021-03-12T00:00:00Z",
            "end": "2021-03-15T00:00:00Z",
            "intervals": ["2021-03-12T00:00:00Z", "2021-03-13T00:00:00Z", "2021-03-14T00:00:00Z"],
            "groups": [
                {
                    "by": {
                        "outcome": "rate_limited",
                        "reason": "spike_protection",
                        "category": "attachment",
                    },
                    "totals": {"sum(quantity)": 1024},
                    "series": {"sum(quantity)": [0, 0, 1024]},
                },
                {
                    "by": {"outcome": "accepted", "reason": "none", "category": "error"},
                    "totals": {"sum(quantity)": 6},
                    "series": {"sum(quantity)": [0, 0, 6]},
                },
                {
                    "by": {
                        "category": "transaction",
                        "reason": "spike_protection",
                        "outcome": "rate_limited",
                    },
                    "totals": {"sum(quantity)": 1},
                    "series": {"sum(quantity)": [0, 0, 1]},
                },
            ],
        }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_staff_org_individual_category(self):
        staff_user = self.create_user(is_staff=True, is_superuser=True)
        self.login_as(user=staff_user, superuser=True)

        category_group_mapping = {
            "attachment": {
                "by": {
                    "outcome": "rate_limited",
                    "reason": "spike_protection",
                },
                "totals": {"sum(quantity)": 1024},
                "series": {"sum(quantity)": [0, 0, 1024]},
            },
            "error": {
                "by": {"outcome": "accepted", "reason": "none"},
                "totals": {"sum(quantity)": 6},
                "series": {"sum(quantity)": [0, 0, 6]},
            },
            "transaction": {
                "by": {
                    "reason": "spike_protection",
                    "outcome": "rate_limited",
                },
                "totals": {"sum(quantity)": 1},
                "series": {"sum(quantity)": [0, 0, 1]},
            },
        }

        # Test each category individually
        for category in ["attachment", "error", "transaction"]:
            response = self.do_request(
                {
                    "category": category,
                    "statsPeriod": "2d",
                    "interval": "1d",
                    "field": ["sum(quantity)"],
                    "groupBy": ["outcome", "reason"],
                },
                org=self.org,
                status_code=200,
            )

            assert result_sorted(response.data) == {
                "start": "2021-03-12T00:00:00Z",
                "end": "2021-03-15T00:00:00Z",
                "intervals": [
                    "2021-03-12T00:00:00Z",
                    "2021-03-13T00:00:00Z",
                    "2021-03-14T00:00:00Z",
                ],
                "groups": [category_group_mapping[category]],
            }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_org_multiple_fields(self):
        response = self.do_request(
            {
                "statsPeriod": "2d",
                "interval": "1d",
                "field": ["sum(quantity)", "sum(times_seen)"],
                "groupBy": ["category", "outcome", "reason"],
            },
            org=self.org,
            status_code=200,
        )

        assert result_sorted(response.data) == {
            "start": "2021-03-12T00:00:00Z",
            "end": "2021-03-15T00:00:00Z",
            "intervals": ["2021-03-12T00:00:00Z", "2021-03-13T00:00:00Z", "2021-03-14T00:00:00Z"],
            "groups": [
                {
                    "by": {
                        "outcome": "rate_limited",
                        "category": "attachment",
                        "reason": "spike_protection",
                    },
                    "totals": {"sum(quantity)": 1024, "sum(times_seen)": 1},
                    "series": {"sum(quantity)": [0, 0, 1024], "sum(times_seen)": [0, 0, 1]},
                },
                {
                    "by": {"outcome": "accepted", "reason": "none", "category": "error"},
                    "totals": {"sum(quantity)": 6, "sum(times_seen)": 6},
                    "series": {"sum(quantity)": [0, 0, 6], "sum(times_seen)": [0, 0, 6]},
                },
                {
                    "by": {
                        "category": "transaction",
                        "reason": "spike_protection",
                        "outcome": "rate_limited",
                    },
                    "totals": {"sum(quantity)": 1, "sum(times_seen)": 1},
                    "series": {"sum(quantity)": [0, 0, 1], "sum(times_seen)": [0, 0, 1]},
                },
            ],
        }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_org_group_by_project(self):
        response = self.do_request(
            {
                "statsPeriod": "1d",
                "interval": "1d",
                "field": ["sum(times_seen)"],
                "groupBy": ["project"],
                "category": ["error", "transaction"],
            },
            org=self.org,
            status_code=200,
        )

        assert result_sorted(response.data) == {
            "start": "2021-03-13T00:00:00Z",
            "end": "2021-03-15T00:00:00Z",
            "groups": [
                {
                    "by": {"project": self.project.id},
                    "totals": {"sum(times_seen)": 6},
                },
                {
                    "by": {"project": self.project2.id},
                    "totals": {"sum(times_seen)": 1},
                },
            ],
        }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_org_project_totals_per_project(self):
        response_per_group = self.do_request(
            {
                "statsPeriod": "1d",
                "interval": "1h",
                "field": ["sum(times_seen)"],
                "groupBy": ["project"],
                "category": ["error", "transaction"],
            },
            org=self.org,
            status_code=200,
        )
        response_total = self.do_request(
            {
                "statsPeriod": "1d",
                "interval": "1h",
                "field": ["sum(times_seen)"],
                "category": ["error", "transaction"],
            },
            org=self.org,
            status_code=200,
        )

        per_group_total = 0
        for total in response_per_group.data["groups"]:
            per_group_total += total["totals"]["sum(times_seen)"]

        assert response_per_group.status_code == 200, response_per_group.content
        assert response_total.status_code == 200, response_total.content
        assert response_total.data["groups"][0]["totals"]["sum(times_seen)"] == per_group_total

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_project_filter(self):
        response = self.do_request(
            {
                "project": self.project.id,
                "statsPeriod": "1d",
                "interval": "1d",
                "field": ["sum(quantity)"],
                "category": ["error", "transaction"],
            },
            org=self.org,
            status_code=200,
        )

        assert result_sorted(response.data) == {
            "start": "2021-03-13T00:00:00Z",
            "end": "2021-03-15T00:00:00Z",
            "intervals": ["2021-03-13T00:00:00Z", "2021-03-14T00:00:00Z"],
            "groups": [
                {"by": {}, "totals": {"sum(quantity)": 6}, "series": {"sum(quantity)": [0, 6]}}
            ],
        }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_staff_project_filter(self):
        staff_user = self.create_user(is_staff=True, is_superuser=True)
        self.login_as(user=staff_user, superuser=True)

        shared_query_params = {
            "field": "sum(quantity)",
            "groupBy": ["outcome", "reason"],
            "interval": "1d",
            "statsPeriod": "1d",
        }
        shared_data = {
            "start": "2021-03-13T00:00:00Z",
            "end": "2021-03-15T00:00:00Z",
            "intervals": ["2021-03-13T00:00:00Z", "2021-03-14T00:00:00Z"],
        }

        # Test error category
        response = self.do_request(
            {
                **shared_query_params,
                "category": "error",
                "project": self.project.id,
            },
            org=self.org,
            status_code=200,
        )

        assert result_sorted(response.data) == {
            **shared_data,
            "groups": [
                {
                    "by": {"outcome": "accepted", "reason": "none"},
                    "totals": {"sum(quantity)": 6},
                    "series": {"sum(quantity)": [0, 6]},
                },
            ],
        }

        # Test transaction category
        response = self.do_request(
            {
                **shared_query_params,
                "category": "transaction",
                "project": self.project2.id,
            },
            org=self.org,
            status_code=200,
        )

        assert result_sorted(response.data) == {
            **shared_data,
            "groups": [
                {
                    "by": {"outcome": "rate_limited", "reason": "spike_protection"},
                    "totals": {"sum(quantity)": 1},
                    "series": {"sum(quantity)": [0, 1]},
                }
            ],
        }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_reason_filter(self):
        response = self.do_request(
            {
                "statsPeriod": "1d",
                "interval": "1d",
                "field": ["sum(times_seen)"],
                "reason": ["spike_protection"],
                "groupBy": ["category"],
            },
            org=self.org,
            status_code=200,
        )

        assert result_sorted(response.data) == {
            "start": "2021-03-13T00:00:00Z",
            "end": "2021-03-15T00:00:00Z",
            "intervals": ["2021-03-13T00:00:00Z", "2021-03-14T00:00:00Z"],
            "groups": [
                {
                    "by": {"category": "attachment"},
                    "totals": {"sum(times_seen)": 1},
                    "series": {"sum(times_seen)": [0, 1]},
                },
                {
                    "by": {"category": "transaction"},
                    "totals": {"sum(times_seen)": 1},
                    "series": {"sum(times_seen)": [0, 1]},
                },
            ],
        }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_outcome_filter(self):
        response = self.do_request(
            {
                "statsPeriod": "1d",
                "interval": "1d",
                "field": ["sum(quantity)"],
                "outcome": "accepted",
                "category": ["error", "transaction"],
            },
            org=self.org,
            status_code=200,
        )

        assert result_sorted(response.data) == {
            "start": "2021-03-13T00:00:00Z",
            "end": "2021-03-15T00:00:00Z",
            "intervals": ["2021-03-13T00:00:00Z", "2021-03-14T00:00:00Z"],
            "groups": [
                {"by": {}, "totals": {"sum(quantity)": 6}, "series": {"sum(quantity)": [0, 6]}}
            ],
        }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_category_filter(self):
        response = self.do_request(
            {
                "statsPeriod": "1d",
                "interval": "1d",
                "field": ["sum(quantity)"],
                "category": "error",
            },
            org=self.org,
            status_code=200,
        )

        assert result_sorted(response.data) == {
            "start": "2021-03-13T00:00:00Z",
            "end": "2021-03-15T00:00:00Z",
            "intervals": ["2021-03-13T00:00:00Z", "2021-03-14T00:00:00Z"],
            "groups": [
                {"by": {}, "totals": {"sum(quantity)": 6}, "series": {"sum(quantity)": [0, 6]}}
            ],
        }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_minute_interval_sum_quantity(self):
        response = self.do_request(
            {
                "statsPeriod": "1h",
                "interval": "15m",
                "field": ["sum(quantity)"],
                "category": "error",
            },
            org=self.org,
            status_code=200,
        )

        assert result_sorted(response.data) == {
            "start": "2021-03-14T11:15:00Z",
            "end": "2021-03-14T12:30:00Z",
            "intervals": [
                "2021-03-14T11:15:00Z",
                "2021-03-14T11:30:00Z",
                "2021-03-14T11:45:00Z",
                "2021-03-14T12:00:00Z",
                "2021-03-14T12:15:00Z",
            ],
            "groups": [
                {
                    "by": {},
                    "totals": {"sum(quantity)": 6},
                    "series": {"sum(quantity)": [6, 0, 0, 0, 0]},
                }
            ],
        }

    @freeze_time("2021-03-14T12:27:28.303Z")
    def test_minute_interval_sum_times_seen(self):
        response = self.do_request(
            {
                "statsPeriod": "1h",
                "interval": "15m",
                "field": ["sum(times_seen)"],
                "category": "error",
            }
        )
        assert response.status_code == 200, response.content
        assert result_sorted(response.data) == {
            "start": "2021-03-14T11:15:00Z",
            "end": "2021-03-14T12:30:00Z",
            "intervals": [
                "2021-03-14T11:15:00Z",
                "2021-03-14T11:30:00Z",
                "2021-03-14T11:45:00Z",
                "2021-03-14T12:00:00Z",
                "2021-03-14T12:15:00Z",
            ],
            "groups": [
                {
                    "by": {},
                    "totals": {"sum(times_seen)": 6},
                    "series": {"sum(times_seen)": [6, 0, 0, 0, 0]},
                }
            ],
        }


def result_sorted(result):
    """sort the groups of the results array by the `by` object, ensuring a stable order"""

    def stable_dict(d):
        return tuple(sorted(d.items(), key=lambda t: t[0]))

    if "groups" in result:
        result["groups"].sort(key=lambda group: stable_dict(group["by"]))
    return result


# TEST invalid parameter