ref(sampling): Create sampling audit logs - (#38501)

src/sentry/api/endpoints/project_details.py

@@ -451,6 +451,9 @@ class ProjectDetailsEndpoint(ProjectEndpoint):
         :param int digestsMaxDelay:
         :auth: required
         """
+
+        old_data = serialize(project, request.user, DetailedProjectSerializer())
+
         has_project_write = request.access and request.access.has_scope("project:write")
 
         changed_proj_settings = {}
@@ -555,7 +558,6 @@ class ProjectDetailsEndpoint(ProjectEndpoint):
                 changed_proj_settings["sentry:secondary_grouping_config"] = result[
                     "secondaryGroupingConfig"
                 ]
-
         if result.get("secondaryGroupingExpiry") is not None:
             if project.update_option(
                 "sentry:secondary_grouping_expiry", result["secondaryGroupingExpiry"]
@@ -657,7 +659,6 @@ class ProjectDetailsEndpoint(ProjectEndpoint):
                 changed_proj_settings["sentry:performance_issue_creation_rate"] = result[
                     "performanceIssueCreationRate"
                 ]
-
         # TODO(dcramer): rewrite options to use standard API config
         if has_project_write:
             options = request.data.get("options", {})
@@ -764,13 +765,24 @@ class ProjectDetailsEndpoint(ProjectEndpoint):
                 if not project.copy_settings_from(result["copy_from_project"]):
                     return Response({"detail": ["Copy project settings failed."]}, status=409)
 
-            self.create_audit_entry(
-                request=request,
-                organization=project.organization,
-                target_object=project.id,
-                event=audit_log.get_event_id("PROJECT_EDIT"),
-                data=changed_proj_settings,
-            )
+            if "sentry:dynamic_sampling" in changed_proj_settings:
+                self.dynamic_sampling_audit_log(
+                    project,
+                    request,
+                    old_data.get("dynamicSampling"),
+                    result.get("dynamicSampling"),
+                )
+                if len(changed_proj_settings) == 1:
+                    data = serialize(project, request.user, DetailedProjectSerializer())
+                    return Response(data)
+
+        self.create_audit_entry(
+            request=request,
+            organization=project.organization,
+            target_object=project.id,
+            event=audit_log.get_event_id("PROJECT_EDIT"),
+            data={**changed_proj_settings, **project.get_audit_log_data()},
+        )
 
         data = serialize(project, request.user, DetailedProjectSerializer())
         return Response(data)
@@ -859,9 +871,73 @@ class ProjectDetailsEndpoint(ProjectEndpoint):
         raw_dynamic_sampling["next_id"] = next_id
         return raw_dynamic_sampling
 
-    def _dynamic_sampling_contains_error_rule(self, raw_dynamic_sampling):
-        if raw_dynamic_sampling is not None:
-            rules = raw_dynamic_sampling.get("rules", [])
-            for rule in rules:
-                if rule["type"] == "error":
-                    return True
+    def dynamic_sampling_audit_log(
+        self, project, request, old_raw_dynamic_sampling, new_raw_dynamic_sampling
+    ):
+        """
+        Compares the previous and next dynamic sampling object, triggering audit logs according to the changes and early returns.
+
+        We are currently verifying the following cases:
+
+        Creation
+            Triggered when the next dynamic sampling object contains more rules than the previous
+
+        Deletion
+            Triggered when the next dynamic sampling object contains less rules than the previous
+
+        Activation
+            We make a loop through the whole object, comparing next with previous rules.
+            If we detect that the rule is different from the another and that the next rule is positive, this is triggered
+
+        Deactivation
+            We make a loop through the whole object, comparing next with previous rules.
+            If we detect that the rule is different from the another and that the next rule is negative, this is triggered
+
+        Other Changes
+            Triggered when all other changes have been made to the next dynamic sampling object
+
+        :old_raw_dynamic_sampling: The dynamic sampling object before the changes
+        :new_raw_dynamic_sampling: The updated dynamic sampling object
+
+        """
+
+        common_audit_data = {
+            "request": request,
+            "organization": project.organization,
+            "target_object": project.id,
+            "data": project.get_audit_log_data(),
+        }
+
+        def create_audit_entry_for_event(audit_data, event_text):
+            self.create_audit_entry(**audit_data, event=audit_log.get_event_id(event_text))
+
+        if old_raw_dynamic_sampling is None:
+            if new_raw_dynamic_sampling is not None:
+                create_audit_entry_for_event(common_audit_data, "SAMPLING_RULE_ADD")
+            return
+
+        old_rules = old_raw_dynamic_sampling.get("rules", [])
+        new_rules = new_raw_dynamic_sampling.get("rules", [])
+
+        if len(new_rules) > len(old_rules):
+            create_audit_entry_for_event(common_audit_data, "SAMPLING_RULE_ADD")
+            return
+
+        if len(new_rules) < len(old_rules):
+            create_audit_entry_for_event(common_audit_data, "SAMPLING_RULE_REMOVE")
+            return
+
+        for index, rule in enumerate(new_rules):
+            if rule["active"] != old_rules[index]["active"]:
+                create_audit_entry_for_event(
+                    common_audit_data,
+                    "SAMPLING_RULE_ACTIVATE" if rule["active"] else "SAMPLING_RULE_DEACTIVATE",
+                )
+                return
+
+        common_audit_data["data"].update(new_raw_dynamic_sampling)
+
+        create_audit_entry_for_event(
+            common_audit_data,
+            "SAMPLING_RULE_EDIT",
+        )

src/sentry/api/serializers/models/project.py

@@ -912,6 +912,15 @@ class DetailedProjectSerializer(ProjectWithTeamSerializer):
 
         return data
 
+    def get_audit_log_data(self):
+        return {
+            "id": self.id,
+            "slug": self.slug,
+            "name": self.name,
+            "status": self.status,
+            "public": self.public,
+        }
+
 
 class SharedProjectSerializer(Serializer):
     def serialize(self, obj, attrs, user):

src/sentry/audit_log/register.py

@@ -301,3 +301,44 @@ default_manager.add(
         template='removed metric alert rule "{label}"',
     )
 )
+
+default_manager.add(
+    AuditLogEvent(
+        event_id=180,
+        name="SAMPLING_RULE_REMOVE",
+        api_name="sampling.remove",
+        template="deleted server-side sampling rule",
+    )
+)
+default_manager.add(
+    AuditLogEvent(
+        event_id=182,
+        name="SAMPLING_RULE_ADD",
+        api_name="sampling.add",
+        template="added server-side sampling rule",
+    )
+)
+default_manager.add(
+    AuditLogEvent(
+        event_id=183,
+        name="SAMPLING_RULE_EDIT",
+        api_name="sampling.edit",
+        template="edited server-side sampling rule",
+    )
+)
+default_manager.add(
+    AuditLogEvent(
+        event_id=184,
+        name="SAMPLING_RULE_ACTIVATE",
+        api_name="sampling.activate",
+        template="activated server-side sampling rule",
+    )
+)
+default_manager.add(
+    AuditLogEvent(
+        event_id=185,
+        name="SAMPLING_RULE_DEACTIVATE",
+        api_name="sampling.deactivate",
+        template="deactivated server-side sampling rule",
+    )
+)

tests/acceptance/test_project_settings_sampling.py

@@ -7,9 +7,10 @@ from django.conf import settings
 from selenium.webdriver.common.action_chains import ActionChains
 from selenium.webdriver.common.keys import Keys
 
+from sentry import audit_log
 from sentry.api.endpoints.project_details import DynamicSamplingSerializer
 from sentry.constants import DataCategory
-from sentry.models import ProjectOption
+from sentry.models import AuditLogEntry, ProjectOption
 from sentry.testutils import AcceptanceTestCase
 from sentry.testutils.skips import requires_snuba
 from sentry.utils import json
@@ -192,6 +193,88 @@ class ProjectSettingsSamplingTest(AcceptanceTestCase):
             assert saved_sampling_setting == serializer.validated_data
             assert uniform_rule_with_custom_sampling_values == serializer.validated_data["rules"][0]
 
+            # Validate the audit log
+            audit_entry = AuditLogEntry.objects.get(
+                organization=self.org, event=audit_log.get_event_id("SAMPLING_RULE_ADD")
+            )
+            audit_log_event = audit_log.get(audit_entry.event)
+            assert audit_log_event.render(audit_entry) == "added server-side sampling rule"
+
+            # Make sure that the early return logic worked, as only the above audit log was triggered
+            with pytest.raises(AuditLogEntry.DoesNotExist):
+                AuditLogEntry.objects.get(
+                    organization=self.org,
+                    target_object=self.project.id,
+                    event=audit_log.get_event_id("PROJECT_EDIT"),
+                )
+
+    def test_remove_specific_rule(self):
+        with self.feature(FEATURE_NAME):
+            self.project.update_option(
+                "sentry:dynamic_sampling",
+                {
+                    "next_id": 3,
+                    "rules": [
+                        {
+                            **specific_rule_with_all_current_trace_conditions,
+                            "id": 1,
+                            "condition": {
+                                "op": "and",
+                                "inner": [
+                                    {
+                                        "op": "glob",
+                                        "name": "trace.release",
+                                        "value": ["[13].[19]"],
+                                    }
+                                ],
+                            },
+                        },
+                        {
+                            **uniform_rule_with_recommended_sampling_values,
+                            "id": 2,
+                        },
+                    ],
+                },
+            )
+
+            self.wait_until_page_loaded()
+
+            action = ActionChains(self.browser.driver)
+
+            # Click on action button
+            action_buttons = self.browser.elements('[aria-label="Actions"]')
+            action.click(action_buttons[0])
+            action.perform()
+
+            # Click on delete button
+            delete_buttons = self.browser.elements('[aria-label="Delete"]')
+            action.click(delete_buttons[0])
+            action.perform()
+
+            # Click on confirm button
+            action.click(self.browser.element('[aria-label="Confirm"]'))
+            action.perform()
+
+            # Wait the success message to show up
+            self.browser.wait_until('[data-test-id="toast-success"]')
+
+            # Validate the audit log
+            audit_entry = AuditLogEntry.objects.get(
+                organization=self.org,
+                event=audit_log.get_event_id("SAMPLING_RULE_REMOVE"),
+                target_object=self.project.id,
+            )
+            audit_log_event = audit_log.get(audit_entry.event)
+            assert audit_log_event.render(audit_entry) == "deleted server-side sampling rule"
+
+            # Make sure that the early return logic worked, as only the above audit log was triggered
+            with pytest.raises(AuditLogEntry.DoesNotExist):
+                AuditLogEntry.objects.get(
+                    organization=self.org,
+                    target_object=self.project.id,
+                    event=audit_log.get_event_id("PROJECT_EDIT"),
+                )
+
     def test_activate_uniform_rule(self):
         with self.feature(FEATURE_NAME):
             self.project.update_option(
@@ -226,6 +309,21 @@ class ProjectSettingsSamplingTest(AcceptanceTestCase):
                 "id": 2,
             } == serializer.validated_data["rules"][0]
 
+            # Validate the audit log
+            audit_entry = AuditLogEntry.objects.get(
+                organization=self.org, event=audit_log.get_event_id("SAMPLING_RULE_ACTIVATE")
+            )
+            audit_log_event = audit_log.get(audit_entry.event)
+            assert audit_log_event.render(audit_entry) == "activated server-side sampling rule"
+
+            # Make sure that the early return logic worked, as only the above audit log was triggered
+            with pytest.raises(AuditLogEntry.DoesNotExist):
+                AuditLogEntry.objects.get(
+                    organization=self.org,
+                    target_object=self.project.id,
+                    event=audit_log.get_event_id("PROJECT_EDIT"),
+                )
+
     def test_deactivate_uniform_rule(self):
         with self.feature(FEATURE_NAME):
             self.project.update_option(
@@ -260,6 +358,21 @@ class ProjectSettingsSamplingTest(AcceptanceTestCase):
                 "id": 2,
             } == serializer.validated_data["rules"][0]
 
+            # Validate the audit log
+            audit_entry = AuditLogEntry.objects.get(
+                organization=self.org, event=audit_log.get_event_id("SAMPLING_RULE_DEACTIVATE")
+            )
+            audit_log_event = audit_log.get(audit_entry.event)
+            assert audit_log_event.render(audit_entry) == "deactivated server-side sampling rule"
+
+            # Make sure that the early return logic worked, as only the above audit log was triggered
+            with pytest.raises(AuditLogEntry.DoesNotExist):
+                AuditLogEntry.objects.get(
+                    organization=self.org,
+                    target_object=self.project.id,
+                    event=audit_log.get_event_id("PROJECT_EDIT"),
+                )
+
     def test_add_specific_rule(self):
         with self.feature(FEATURE_NAME):
             self.project.update_option(
@@ -425,3 +538,18 @@ class ProjectSettingsSamplingTest(AcceptanceTestCase):
             rulesAfter = self.browser.elements('[data-test-id="sampling-rule"]')
             assert "Environment" in rulesAfter[0].text
             assert "Release" in rulesAfter[1].text
+
+            # Validate the audit log
+            audit_entry = AuditLogEntry.objects.get(
+                organization=self.org, event=audit_log.get_event_id("SAMPLING_RULE_EDIT")
+            )
+            audit_log_event = audit_log.get(audit_entry.event)
+            assert audit_log_event.render(audit_entry) == "edited server-side sampling rule"
+
+            # Make sure that the early return logic worked, as only the above audit log was triggered
+            with pytest.raises(AuditLogEntry.DoesNotExist):
+                AuditLogEntry.objects.get(
+                    organization=self.org,
+                    target_object=self.project.id,
+                    event=audit_log.get_event_id("PROJECT_EDIT"),
+                )

tests/sentry/api/endpoints/test_project_details.py

@@ -800,6 +800,261 @@ class ProjectUpdateTest(APITestCase):
         _remove_ids_from_dynamic_rules(saved_config)
         _remove_ids_from_dynamic_rules(original_config)
         assert original_config == saved_config
+        assert AuditLogEntry.objects.filter(
+            organization=self.project.organization,
+            event=audit_log.get_event_id("SAMPLING_RULE_ADD"),
+        ).exists()
+
+        # Make sure that the early return logic worked, as only the above audit log was triggered
+        with pytest.raises(AuditLogEntry.DoesNotExist):
+            AuditLogEntry.objects.get(
+                organization=self.organization,
+                target_object=self.project.id,
+                event=audit_log.get_event_id("PROJECT_EDIT"),
+            )
+
+    def test_dynamic_sampling_rule_deletion(self):
+        """
+        Tests that when sending a request to dekete a dynamic sampling rule,
+        the rule will be successfully deleted and that the audit log 'SAMPLING_RULE_REMOVE' will be triggered
+        """
+        dynamic_sampling = _dyn_sampling_data()
+        project = self.project  # force creation
+        # Update project adding three rules
+        project.update_option("sentry:dynamic_sampling", dynamic_sampling)
+
+        self.login_as(self.user)
+
+        token = ApiToken.objects.create(user=self.user, scope_list=["project:write"])
+        authorization = f"Bearer {token.token}"
+
+        url = reverse(
+            "sentry-api-0-project-details",
+            kwargs={
+                "organization_slug": self.project.organization.slug,
+                "project_slug": self.project.slug,
+            },
+        )
+
+        data = {
+            "dynamicSampling": {
+                "rules": [dynamic_sampling["rules"][2]],
+            }
+        }
+
+        with Feature({"organizations:server-side-sampling": True}):
+            self.client.put(url, format="json", HTTP_AUTHORIZATION=authorization, data=data)
+
+            assert AuditLogEntry.objects.filter(
+                organization=self.project.organization,
+                event=audit_log.get_event_id("SAMPLING_RULE_REMOVE"),
+            ).exists()
+
+            # Make sure that the early return logic worked, as only the above audit log was triggered
+            with pytest.raises(AuditLogEntry.DoesNotExist):
+                AuditLogEntry.objects.get(
+                    organization=self.organization,
+                    target_object=self.project.id,
+                    event=audit_log.get_event_id("PROJECT_EDIT"),
+                )
+
+    def test_dynamic_sampling_rule_activation(self):
+        """
+        Tests that when sending a request to activate a dynamic sampling rule,
+        the rule will be successfully activated and that the audit log 'SAMPLING_RULE_ACTIVATE' will be triggered
+        """
+        dynamic_sampling = _dyn_sampling_data()
+        project = self.project  # force creation
+        # Update project adding three rules
+        project.update_option(
+            "sentry:dynamic_sampling",
+            {
+                "rules": [
+                    {**dynamic_sampling["rules"][1], "active": False},
+                    {**dynamic_sampling["rules"][2], "active": False},
+                ]
+            },
+        )
+
+        self.login_as(self.user)
+
+        token = ApiToken.objects.create(user=self.user, scope_list=["project:write"])
+        authorization = f"Bearer {token.token}"
+
+        url = reverse(
+            "sentry-api-0-project-details",
+            kwargs={
+                "organization_slug": self.project.organization.slug,
+                "project_slug": self.project.slug,
+            },
+        )
+
+        data = {
+            "dynamicSampling": {
+                "rules": [
+                    {**dynamic_sampling["rules"][1], "active": False},
+                    {**dynamic_sampling["rules"][2], "active": True},
+                ]
+            }
+        }
+
+        with Feature({"organizations:server-side-sampling": True}):
+            self.client.put(url, format="json", HTTP_AUTHORIZATION=authorization, data=data)
+
+            assert AuditLogEntry.objects.filter(
+                organization=self.project.organization,
+                event=audit_log.get_event_id("SAMPLING_RULE_ACTIVATE"),
+            ).exists()
+
+            # Make sure that the early return logic worked, as only the above audit log was triggered
+            with pytest.raises(AuditLogEntry.DoesNotExist):
+                AuditLogEntry.objects.get(
+                    organization=self.organization,
+                    target_object=self.project.id,
+                    event=audit_log.get_event_id("PROJECT_EDIT"),
+                )
+
+    def test_dynamic_sampling_rule_deactivation(self):
+        """
+        Tests that when sending a request to deactivate a dynamic sampling rule,
+        the rule will be successfully deactivated and that the audit log 'SAMPLING_RULE_DEACTIVATE' will be triggered
+        """
+        dynamic_sampling = _dyn_sampling_data()
+        project = self.project  # force creation
+        # Update project adding three rules
+        project.update_option("sentry:dynamic_sampling", dynamic_sampling)
+
+        self.login_as(self.user)
+
+        token = ApiToken.objects.create(user=self.user, scope_list=["project:write"])
+        authorization = f"Bearer {token.token}"
+
+        url = reverse(
+            "sentry-api-0-project-details",
+            kwargs={
+                "organization_slug": self.project.organization.slug,
+                "project_slug": self.project.slug,
+            },
+        )
+
+        data = {
+            "dynamicSampling": {
+                "rules": [
+                    {**dynamic_sampling["rules"][0], "active": False},
+                    dynamic_sampling["rules"][1],
+                    dynamic_sampling["rules"][2],
+                ]
+            }
+        }
+
+        with Feature({"organizations:server-side-sampling": True}):
+            self.client.put(url, format="json", HTTP_AUTHORIZATION=authorization, data=data)
+
+            assert AuditLogEntry.objects.filter(
+                organization=self.project.organization,
+                event=audit_log.get_event_id("SAMPLING_RULE_DEACTIVATE"),
+            ).exists()
+
+            # Make sure that the early return logic worked, as only the above audit log was triggered
+            with pytest.raises(AuditLogEntry.DoesNotExist):
+                AuditLogEntry.objects.get(
+                    organization=self.organization,
+                    target_object=self.project.id,
+                    event=audit_log.get_event_id("PROJECT_EDIT"),
+                )
+
+    def test_dynamic_smapling_rule_edition(self):
+        """
+        Tests that when sending a request updating a dynamic sampling rule,
+        the rule will be successfully edited and that the audit log 'SAMPLING_RULE_EDIT' will be triggered
+        """
+        dynamic_sampling = _dyn_sampling_data()
+        project = self.project  # force creation
+        # Update project adding three rules
+        project.update_option("sentry:dynamic_sampling", dynamic_sampling)
+
+        self.login_as(self.user)
+
+        token = ApiToken.objects.create(user=self.user, scope_list=["project:write"])
+        authorization = f"Bearer {token.token}"
+
+        url = reverse(
+            "sentry-api-0-project-details",
+            kwargs={
+                "organization_slug": self.project.organization.slug,
+                "project_slug": self.project.slug,
+            },
+        )
+
+        data = {
+            "dynamicSampling": {
+                "rules": [
+                    {**dynamic_sampling["rules"][0], "sampleRate": 0.2},
+                    dynamic_sampling["rules"][1],
+                    dynamic_sampling["rules"][2],
+                ]
+            }
+        }
+
+        with Feature({"organizations:server-side-sampling": True}):
+            self.client.put(url, format="json", HTTP_AUTHORIZATION=authorization, data=data)
+
+            assert AuditLogEntry.objects.filter(
+                organization=self.project.organization,
+                event=audit_log.get_event_id("SAMPLING_RULE_EDIT"),
+            ).exists()
+
+            # Make sure that the early return logic worked, as only the above audit log was triggered
+            with pytest.raises(AuditLogEntry.DoesNotExist):
+                AuditLogEntry.objects.get(
+                    organization=self.organization,
+                    target_object=self.project.id,
+                    event=audit_log.get_event_id("PROJECT_EDIT"),
+                )
+
+    def test_request_with_dynamic_sampling_and_other_property(self):
+        """
+        Tests that when sending a request to update the dynamic sampling property
+        alongside another project's property, everything will be successfully updated and
+        the audit logs 'SAMPLING_RULE_*' and 'PROJECT_EDIT' will be triggered
+
+        """
+
+        dynamic_sampling = _dyn_sampling_data()
+        self.login_as(self.user)
+
+        token = ApiToken.objects.create(user=self.user, scope_list=["project:write"])
+        authorization = f"Bearer {token.token}"
+
+        url = reverse(
+            "sentry-api-0-project-details",
+            kwargs={
+                "organization_slug": self.project.organization.slug,
+                "project_slug": self.project.slug,
+            },
+        )
+
+        data = {
+            "dynamicSampling": {
+                "rules": [dynamic_sampling["rules"][len(dynamic_sampling["rules"]) - 1]],
+            },
+            "platform": "rust",
+            "relayPiiConfig": "",
+        }
+
+        with Feature({"organizations:server-side-sampling": True}):
+            self.client.put(url, format="json", HTTP_AUTHORIZATION=authorization, data=data)
+
+            # Audit Log shall be triggered twice
+            assert AuditLogEntry.objects.filter(
+                organization=self.project.organization,
+                event=audit_log.get_event_id("SAMPLING_RULE_ADD"),
+            ).exists()
+
+            assert AuditLogEntry.objects.filter(
+                organization=self.project.organization,
+                event=audit_log.get_event_id("PROJECT_EDIT"),
+            ).exists()
 
     def test_setting_dynamic_sampling_rules_roundtrip(self):
         """
@@ -927,6 +1182,19 @@ class ProjectUpdateTest(APITestCase):
             response = self.get_success_response(self.org_slug, self.proj_slug, method="get")
         saved_config = response.data["dynamicSampling"]
         assert all([rule["active"] for rule in saved_config["rules"]])
+        assert AuditLogEntry.objects.filter(
+            organization=self.project.organization,
+            target_object=self.project.id,
+            event=audit_log.get_event_id("SAMPLING_RULE_ADD"),
+        ).exists()
+
+        # Make sure that the early return logic worked, as only the above audit log was triggered
+        with pytest.raises(AuditLogEntry.DoesNotExist):
+            AuditLogEntry.objects.get(
+                organization=self.organization,
+                target_object=self.project.id,
+                event=audit_log.get_event_id("PROJECT_EDIT"),
+            )
 
     def test_dynamic_sampling_rules_should_contain_single_uniform_rule(self):
         """
@@ -1031,7 +1299,18 @@ class ProjectUpdateTest(APITestCase):
             # check that audit entry was created with redacted password
             assert create_audit_entry.called
             call = faux.faux(create_audit_entry)
-            assert call.kwarg_equals("data", {"sentry:symbol_sources": [redacted_source]})
+
+            assert call.kwarg_equals(
+                "data",
+                {
+                    "sentry:symbol_sources": [redacted_source],
+                    "id": self.project.id,
+                    "slug": self.project.slug,
+                    "name": self.project.name,
+                    "status": self.project.status,
+                    "public": self.project.public,
+                },
+            )
 
             self.get_success_response(
                 self.org_slug, self.proj_slug, symbolSources=json.dumps([redacted_source])

tests/sentry/audit_log/test_register.py

@@ -53,6 +53,11 @@ class AuditLogEventRegisterTest(TestCase):
             "sentry-app.remove",
             "sentry-app.install",
             "sentry-app.uninstall",
+            "sampling.edit",
+            "sampling.remove",
+            "sampling.deactivate",
+            "sampling.activate",
+            "sampling.add",
             "monitor.add",
             "monitor.edit",
             "monitor.remove",