Add prometheus to docker compose for testing

README.md

@@ -34,8 +34,8 @@ View our [Contributing](./CONTRIBUTING.md) documentation if you'd like to help m
 ## Run local dev environment
 
 1. Ensure docker and docker-compose are installed
-2. `docker-compose up`
-3. `docker-compose run --rm web ./manage.py migrate`
+2. `docker compose up`
+3. `docker compose run --rm web ./manage.py migrate`
 
 Run tests with `docker-compose run --rm web ./manage.py test`
 
@@ -58,12 +58,17 @@ VS Code can do type checking and type inference. However, it requires setting up
 ### Load testing
 
 Locust is built into the dev dependencies. To run with Locust run
-`docker-compose -f docker-compose.yml -f docker-compose.locust.yml up`
+`docker compose -f docker-compose.yml -f docker-compose.locust.yml up`
 
 Now go to localhost:8089 to run the test.
 
 Locust will not be installed to production docker images and cannot be run from them.
 
+### Observability metrics with Prometheus
+
+1. Edit monitoring/prometheus/prometheus.yml and set credentials to a GlitchTip auth token
+2. `docker compose -f docker-compose.yml -f docker-compose.metrics.yml up`
+
 # GCP Logging
 
 In order to enable json logging, set the environment as follows::

docker-compose.metrics.yml

@@ -0,0 +1,19 @@
+version: "3.8"
+
+services:
+  prometheus:
+    image: prom/prometheus
+    volumes:
+      - ./monitoring/prometheus/:/etc/prometheus/
+      - prometheus_data:/prometheus
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+      - "--web.console.templates=/usr/share/prometheus/consoles"
+      - "--web.enable-lifecycle"
+    ports:
+      - 9090:9090
+
+volumes:
+  prometheus_data: {}

docker-compose.yml

@@ -1,4 +1,4 @@
-version: "3.7"
+version: "3.8"
 x-environment: &default-environment
   DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres
   SECRET_KEY: change_me

monitoring/prometheus/django.rules

@@ -0,0 +1,106 @@
+
+groups:
+- name: django.rules
+  rules:
+  - record: job:django_http_requests_before_middlewares_total:sum_rate30s
+    expr: sum(rate(django_http_requests_before_middlewares_total[30s])) BY (job)
+  - record: job:django_http_requests_unknown_latency_total:sum_rate30s
+    expr: sum(rate(django_http_requests_unknown_latency_total[30s])) BY (job)
+  - record: job:django_http_ajax_requests_total:sum_rate30s
+    expr: sum(rate(django_http_ajax_requests_total[30s])) BY (job)
+  - record: job:django_http_responses_before_middlewares_total:sum_rate30s
+    expr: sum(rate(django_http_responses_before_middlewares_total[30s])) BY (job)
+  - record: job:django_http_requests_unknown_latency_including_middlewares_total:sum_rate30s
+    expr: sum(rate(django_http_requests_unknown_latency_including_middlewares_total[30s]))
+      BY (job)
+  - record: job:django_http_requests_body_total_bytes:sum_rate30s
+    expr: sum(rate(django_http_requests_body_total_bytes[30s])) BY (job)
+  - record: job:django_http_responses_streaming_total:sum_rate30s
+    expr: sum(rate(django_http_responses_streaming_total[30s])) BY (job)
+  - record: job:django_http_responses_body_total_bytes:sum_rate30s
+    expr: sum(rate(django_http_responses_body_total_bytes[30s])) BY (job)
+  - record: job:django_http_requests_total:sum_rate30s
+    expr: sum(rate(django_http_requests_total_by_method[30s])) BY (job)
+  - record: job:django_http_requests_total_by_method:sum_rate30s
+    expr: sum(rate(django_http_requests_total_by_method[30s])) BY (job, method)
+  - record: job:django_http_requests_total_by_transport:sum_rate30s
+    expr: sum(rate(django_http_requests_total_by_transport[30s])) BY (job, transport)
+  - record: job:django_http_requests_total_by_view:sum_rate30s
+    expr: sum(rate(django_http_requests_total_by_view_transport_method[30s])) BY (job,
+      view)
+  - record: job:django_http_requests_total_by_view_transport_method:sum_rate30s
+    expr: sum(rate(django_http_requests_total_by_view_transport_method[30s])) BY (job,
+      view, transport, method)
+  - record: job:django_http_responses_total_by_templatename:sum_rate30s
+    expr: sum(rate(django_http_responses_total_by_templatename[30s])) BY (job, templatename)
+  - record: job:django_http_responses_total_by_status:sum_rate30s
+    expr: sum(rate(django_http_responses_total_by_status[30s])) BY (job, status)
+  - record: job:django_http_responses_total_by_status_name_method:sum_rate30s
+    expr: sum(rate(django_http_responses_total_by_status_name_method[30s])) BY (job,
+      status, name, method)
+  - record: job:django_http_responses_total_by_charset:sum_rate30s
+    expr: sum(rate(django_http_responses_total_by_charset[30s])) BY (job, charset)
+  - record: job:django_http_exceptions_total_by_type:sum_rate30s
+    expr: sum(rate(django_http_exceptions_total_by_type[30s])) BY (job, type)
+  - record: job:django_http_exceptions_total_by_view:sum_rate30s
+    expr: sum(rate(django_http_exceptions_total_by_view[30s])) BY (job, view)
+  - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
+    expr: histogram_quantile(0.5, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
+      BY (job, le))
+    labels:
+      quantile: "50"
+  - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
+    expr: histogram_quantile(0.95, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
+      BY (job, le))
+    labels:
+      quantile: "95"
+  - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
+    expr: histogram_quantile(0.99, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
+      BY (job, le))
+    labels:
+      quantile: "99"
+  - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
+    expr: histogram_quantile(0.999, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
+      BY (job, le))
+    labels:
+      quantile: "99.9"
+  - record: job:django_http_requests_latency_seconds:quantile_rate30s
+    expr: histogram_quantile(0.5, sum(rate(django_http_requests_latency_seconds_bucket[30s]))
+      BY (job, le))
+    labels:
+      quantile: "50"
+  - record: job:django_http_requests_latency_seconds:quantile_rate30s
+    expr: histogram_quantile(0.95, sum(rate(django_http_requests_latency_seconds_bucket[30s]))
+      BY (job, le))
+    labels:
+      quantile: "95"
+  - record: job:django_http_requests_latency_seconds:quantile_rate30s
+    expr: histogram_quantile(0.99, sum(rate(django_http_requests_latency_seconds_bucket[30s]))
+      BY (job, le))
+    labels:
+      quantile: "99"
+  - record: job:django_http_requests_latency_seconds:quantile_rate30s
+    expr: histogram_quantile(0.999, sum(rate(django_http_requests_latency_seconds_bucket[30s]))
+      BY (job, le))
+    labels:
+      quantile: "99.9"
+  - record: job:django_model_inserts_total:sum_rate1m
+    expr: sum(rate(django_model_inserts_total[1m])) BY (job, model)
+  - record: job:django_model_updates_total:sum_rate1m
+    expr: sum(rate(django_model_updates_total[1m])) BY (job, model)
+  - record: job:django_model_deletes_total:sum_rate1m
+    expr: sum(rate(django_model_deletes_total[1m])) BY (job, model)
+  - record: job:django_db_new_connections_total:sum_rate30s
+    expr: sum(rate(django_db_new_connections_total[30s])) BY (alias, vendor)
+  - record: job:django_db_new_connection_errors_total:sum_rate30s
+    expr: sum(rate(django_db_new_connection_errors_total[30s])) BY (alias, vendor)
+  - record: job:django_db_execute_total:sum_rate30s
+    expr: sum(rate(django_db_execute_total[30s])) BY (alias, vendor)
+  - record: job:django_db_execute_many_total:sum_rate30s
+    expr: sum(rate(django_db_execute_many_total[30s])) BY (alias, vendor)
+  - record: job:django_db_errors_total:sum_rate30s
+    expr: sum(rate(django_db_errors_total[30s])) BY (alias, vendor, type)
+  - record: job:django_migrations_applied_total:max
+    expr: max(django_migrations_applied_total) BY (job, connection)
+  - record: job:django_migrations_unapplied_total:max
+    expr: max(django_migrations_unapplied_total) BY (job, connection)

monitoring/prometheus/prometheus.yml

@@ -0,0 +1,17 @@
+global:
+  scrape_interval: 10s
+  evaluation_interval: 10s
+
+  external_labels:
+    monitor: django-monitor
+
+rule_files:
+  - "django.rules"
+
+scrape_configs:
+  - job_name: "glitchtip-backend"
+    static_configs:
+      - targets: ["web:8000"]
+    metrics_path: /api/0/observability/django/
+    authorization:
+      credentials: enter-auth-token-here

poetry.lock

@@ -204,14 +204,14 @@ uvloop = ["uvloop (>=0.15.2)"]
 
 [[package]]
 name = "boto3"
-version = "1.24.35"
+version = "1.24.36"
 description = "The AWS SDK for Python"
 category = "main"
 optional = false
 python-versions = ">= 3.7"
 
 [package.dependencies]
-botocore = ">=1.27.35,<1.28.0"
+botocore = ">=1.27.36,<1.28.0"
 jmespath = ">=0.7.1,<2.0.0"
 s3transfer = ">=0.6.0,<0.7.0"
 
@@ -220,7 +220,7 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
 
 [[package]]
 name = "botocore"
-version = "1.27.35"
+version = "1.27.36"
 description = "Low-level, data-driven core of boto 3."
 category = "main"
 optional = false
@@ -852,7 +852,7 @@ validation = ["swagger-spec-validator (>=2.1.0)"]
 
 [[package]]
 name = "executing"
-version = "0.8.3"
+version = "0.9.0"
 description = "Get the currently executing AST node of a frame, and other information"
 category = "dev"
 optional = false
@@ -2122,7 +2122,7 @@ python-versions = "*"
 
 [[package]]
 name = "types-pyyaml"
-version = "6.0.10"
+version = "6.0.11"
 description = "Typing stubs for PyYAML"
 category = "dev"
 optional = false
@@ -2210,12 +2210,15 @@ python-versions = "*"
 
 [[package]]
 name = "werkzeug"
-version = "2.1.2"
+version = "2.2.0"
 description = "The comprehensive WSGI web application library."
 category = "dev"
 optional = false
 python-versions = ">=3.7"
 
+[package.dependencies]
+MarkupSafe = ">=2.1.1"
+
 [package.extras]
 watchdog = ["watchdog"]
 
@@ -2455,12 +2458,12 @@ black = [
     {file = "black-22.6.0.tar.gz", hash = "sha256:6c6d39e28aed379aec40da1c65434c77d75e65bb59a1e1c283de545fb4e7c6c9"},
 ]
 boto3 = [
-    {file = "boto3-1.24.35-py3-none-any.whl", hash = "sha256:1822c1524c5be007b966498cd196a040a1bd414c8aca7b270431a10d3ade8294"},
-    {file = "boto3-1.24.35.tar.gz", hash = "sha256:2bc600c88d963bbf98cb1244e34f5b7c889a861121b95041c6a47f917fa46748"},
+    {file = "boto3-1.24.36-py3-none-any.whl", hash = "sha256:8844bbcb69ac0afc68225b58abe105852231cf1b562e6c8c9eb6b2b97fd4757a"},
+    {file = "boto3-1.24.36.tar.gz", hash = "sha256:b1855ede59e725b968d6336908ffc864b65985ca441d730625b09c43ccd6413b"},
 ]
 botocore = [
-    {file = "botocore-1.27.35-py3-none-any.whl", hash = "sha256:9949d61959476b5a34408881bdb98f54b0642238ffb217c5260124ec58fb0c72"},
-    {file = "botocore-1.27.35.tar.gz", hash = "sha256:d2e708dd766b21c8e20a57ce1a90e98d324f871f81215efbc2dddaa42d13c551"},
+    {file = "botocore-1.27.36-py3-none-any.whl", hash = "sha256:3119ce186053b9bf6bd0bd0ad19a8cedeb626b205ce6ad26ea0894634f702cd5"},
+    {file = "botocore-1.27.36.tar.gz", hash = "sha256:8109526f55742539d2311d742b40c89e65781ad18966e577dda360cd55c9d047"},
 ]
 brotli = [
     {file = "Brotli-1.0.9-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:268fe94547ba25b58ebc724680609c8ee3e5a843202e9a381f6f9c5e8bdb5c70"},
@@ -2781,8 +2784,8 @@ drf-yasg = [
     {file = "drf_yasg-1.21.3-py3-none-any.whl", hash = "sha256:85899b8b173cac7d39af3fea205bdc1a599aed8de3389a3bcccc29b37b9b983e"},
 ]
 executing = [
-    {file = "executing-0.8.3-py2.py3-none-any.whl", hash = "sha256:d1eef132db1b83649a3905ca6dd8897f71ac6f8cac79a7e58a1a09cf137546c9"},
-    {file = "executing-0.8.3.tar.gz", hash = "sha256:c6554e21c6b060590a6d3be4b82fb78f8f0194d809de5ea7df1c093763311501"},
+    {file = "executing-0.9.0-py2.py3-none-any.whl", hash = "sha256:d07e9a46c85dd507055f7c4c208689faef1bf6a671ae81e91787f307808bacfb"},
+    {file = "executing-0.9.0.tar.gz", hash = "sha256:ade7276b4b108df69b8480064264db856335585efe170833601f30bcaaed7bc7"},
 ]
 fido2 = [
     {file = "fido2-1.0.0-py3-none-any.whl", hash = "sha256:dce13d739b8e0df30505b33f5fd2868fad20f3b309acacce72e5f2d1b0c58761"},
@@ -3951,8 +3954,8 @@ types-pytz = [
     {file = "types_pytz-2022.1.2-py3-none-any.whl", hash = "sha256:8aa9fd2af9dee5f5bd7221c6804c9addeafa7ebc0008f544d4ace02b066818a4"},
 ]
 types-pyyaml = [
-    {file = "types-PyYAML-6.0.10.tar.gz", hash = "sha256:a1676caeb098096833fe2f1472e7ea8debf7663222964fa90545cfaafebf3385"},
-    {file = "types_PyYAML-6.0.10-py3-none-any.whl", hash = "sha256:18985a382c93fb12187daae6f57ce7787e3651cd590291c907d3e757919689ff"},
+    {file = "types-PyYAML-6.0.11.tar.gz", hash = "sha256:7f7da2fd11e9bc1e5e9eb3ea1be84f4849747017a59fc2eee0ea34ed1147c2e0"},
+    {file = "types_PyYAML-6.0.11-py3-none-any.whl", hash = "sha256:8f890028123607379c63550179ddaec4517dc751f4c527a52bb61934bf495989"},
 ]
 typing-extensions = [
     {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"},
@@ -3990,8 +3993,8 @@ wcwidth = [
     {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"},
 ]
 werkzeug = [
-    {file = "Werkzeug-2.1.2-py3-none-any.whl", hash = "sha256:72a4b735692dd3135217911cbeaa1be5fa3f62bffb8745c5215420a03dc55255"},
-    {file = "Werkzeug-2.1.2.tar.gz", hash = "sha256:1ce08e8093ed67d638d63879fd1ba3735817f7a80de3674d293f5984f25fb6e6"},
+    {file = "Werkzeug-2.2.0-py3-none-any.whl", hash = "sha256:81806f8a5b35e6cb1b39a6f28dabf0e123f069c8596119a1a9a43838870016cd"},
+    {file = "Werkzeug-2.2.0.tar.gz", hash = "sha256:fe8bcdcef40275ed915fc734c2527a39d705b57a716d4f09e790296abbd16a7f"},
 ]
 whitenoise = [
     {file = "whitenoise-6.2.0-py3-none-any.whl", hash = "sha256:8e9c600a5c18bd17655ef668ad55b5edf6c24ce9bdca5bf607649ca4b1e8e2c2"},