Fix tests, use function to ignore tsvector size limit

events/tests/tests.py

@@ -88,7 +88,7 @@ class EventStoreTestCase(APITestCase):
     def test_performance(self):
         with open("events/test_data/py_hi_event.json") as json_file:
             data = json.load(json_file)
-        with self.assertNumQueries(14):
+        with self.assertNumQueries(16):
             res = self.client.post(self.url, data, format="json")
         self.assertEqual(res.status_code, 200)
 

issues/migrations/0005_issue_tags.py

@@ -3,6 +3,7 @@
 import collections
 from django.db import migrations, models
 from .sql.triggers import UPDATE_ISSUE_TRIGGER
+from .sql.functions import GENERATE_ISSUE_TSVECTOR
 
 
 def forwards_func(apps, schema_editor):
@@ -37,5 +38,6 @@ class Migration(migrations.Migration):
             model_name="issue", name="tags", field=models.JSONField(default=dict),
         ),
         migrations.RunSQL(UPDATE_ISSUE_TRIGGER, UPDATE_ISSUE_TRIGGER),
+        migrations.RunSQL(GENERATE_ISSUE_TSVECTOR, GENERATE_ISSUE_TSVECTOR),
         migrations.RunPython(forwards_func, noop),
     ]

issues/migrations/sql/functions.py

@@ -0,0 +1,9 @@
+GENERATE_ISSUE_TSVECTOR = """
+CREATE OR REPLACE FUNCTION generate_issue_tsvector(jsonb) RETURNS tsvector AS $$
+BEGIN
+    RETURN strip(jsonb_to_tsvector('english', $1, '["string"]'));
+    EXCEPTION WHEN program_limit_exceeded THEN
+    RETURN null;
+END;
+$$ LANGUAGE plpgsql;;
+"""

issues/models.py

@@ -4,6 +4,7 @@ from django.contrib.postgres.indexes import GinIndex
 from django.conf import settings
 from django.db import models
 from django.db.models import Max, Count
+from django.db.utils import InternalError
 from events.models import LogLevel
 from glitchtip.model_utils import FromStringIntegerChoices
 from glitchtip.base_models import CreatedModel
@@ -104,7 +105,7 @@ class Issue(CreatedModel):
         """
         Update search index/tag aggregations
         """
-        vector_query = """SELECT strip(jsonb_to_tsvector('english', jsonb_agg(data), '["string"]')) from (SELECT events_event.data from events_event where issue_id = %s limit 100) as data"""
+        vector_query = """SELECT generate_issue_tsvector(jsonb_agg(data)) from (SELECT events_event.data from events_event where issue_id = %s limit 200) as data"""
         issue = (
             cls.objects.extra(
                 select={"new_vector": vector_query}, select_params=(issue_id,)
@@ -117,12 +118,20 @@ class Issue(CreatedModel):
             .defer("search_vector")
             .get(pk=issue_id)
         )
-        issue.search_vector = issue.new_vector
-        issue.last_seen = issue.new_last_seen
-        issue.count = issue.new_count
-        issue.level = issue.new_level
 
-        update_fields = ["search_vector", "last_seen", "count", "level"]
+        update_fields = ["last_seen", "count", "level"]
+        if (
+            issue.new_vector
+        ):  # This check is important, because generate_issue_tsvector returns null on size limit
+            update_fields.append("search_vector")
+            issue.search_vector = issue.new_vector
+        if issue.new_last_seen:
+            issue.last_seen = issue.new_last_seen
+        if issue.new_count:
+            issue.count = issue.new_count
+        if issue.new_level:
+            issue.level = issue.new_level
+
         if not skip_tags:
             update_fields.append("tags")
             tags = (

issues/tests/test_search_filter.py

@@ -3,6 +3,7 @@ from django.utils import timezone
 from freezegun import freeze_time
 from model_bakery import baker
 from glitchtip.test_utils.test_case import GlitchTipTestCase
+from issues.tasks import update_search_index_all_issues
 from ..tasks import update_search_index_all_issues
 
 
@@ -178,6 +179,8 @@ class SearchTestCase(GlitchTipTestCase):
             "events.Event", issue=event.issue, data={"name": "apple sauce"}
         )
         other_event = baker.make("events.Event", issue__project=self.project)
+        update_search_index_all_issues()
+
         res = self.client.get(self.url + "?query=is:unresolved apple+sauce")
         self.assertContains(res, event.issue.title)
         self.assertNotContains(res, other_event.issue.title)

issues/tests/tests.py

@@ -136,7 +136,7 @@ class IssuesAPITestCase(GlitchTipTestCase):
         events = baker.make("events.Event", issue=issue, _quantity=2)
         res = self.client.get(self.url)
         self.assertEqual(
-            res.data[0]["lastSeen"][:25], events[1].created.isoformat()[:25]
+            res.data[0]["lastSeen"][:20], events[1].created.isoformat()[:20]
         )
 
     def test_issue_delete(self):
@@ -261,6 +261,7 @@ class IssuesAPITestCase(GlitchTipTestCase):
 
         baker.make("events.Event", issue=issue2, _quantity=2)
         baker.make("events.Event", issue=issue1)
+        update_search_index_all_issues()
 
         res = self.client.get(self.url)
         self.assertEqual(res.data[0]["id"], issue1.id)