ci(lint): Add requirements checker (#26822)

This is to prevent incidents like #26632

.github/file-filters.yml

@@ -34,6 +34,7 @@ frontend_components_modified_lintable:
 
 backend_lintable: &backend_lintable
   - '**/*.py'
+  - 'requirements-base.txt'
 
 # Currently used in `getsentry-dispatch.yml` to dispatch backend tests on getsentry
 backend_dependencies: &backend_dependencies

.github/workflows/backend-lint.yml

@@ -89,6 +89,8 @@ jobs:
           filters: |
             all:
               - added|modified: '**/*.py'
+              - added|modified: 'requirements-base.txt'
+
 
       - name: Run pre-commit on changed files
         if: steps.changes.outputs.backend == 'true'

.pre-commit-config.yaml

@@ -51,6 +51,12 @@ repos:
       # We can't change this mock to unittest.mock until Python 3.8,
       # refer to the comment in this file.
       exclude: 'src/sentry/utils/compat/mock.py'
+    - id: lint-requirements
+      name: lint-requirements
+      entry: bin/lint-requirements
+      language: system
+      files: requirements-base.txt
+      pass_filenames: false
 
 -   repo: https://github.com/sirosen/check-jsonschema
     rev: 0.3.0

bin/lint-requirements

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+import sys
+
+import requirements
+
+
+def main():
+    """
+    We cannot have non-specifier requirements if we want to publish to PyPI
+    due to security concerns. This check ensures we don't have/add any URL/VCS
+    dependencies in the base requirements file.
+    """
+    with open("requirements-base.txt") as reqs_file:
+        if any(not req.specifier for req in requirements.parse(reqs_file)):
+            print(
+                "\n".join(
+                    [
+                        "You cannot use dependencies that are not on PyPI directly.",
+                        "See PEP440: https://www.python.org/dev/peps/pep-0440/#direct-references",
+                    ]
+                ),
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

requirements-pre-commit.txt

@@ -3,3 +3,4 @@ black==21.5b1
 sentry-flake8==1.0.0
 pyupgrade==2.18.3
 isort==5.8.0
+requirements-parser>=0.2.0