action.yml 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208
  1. # NOTE: Do not rely on `make` commands here as this action is used across different repos
  2. # where the Makefile will not be available
  3. name: 'Sentry Setup'
  4. description: 'Sets up a Sentry test environment'
  5. inputs:
  6. workdir:
  7. description: 'Directory where the sentry source is located'
  8. required: false
  9. default: '.'
  10. snuba:
  11. description: 'Is snuba required?'
  12. required: false
  13. default: 'false'
  14. clickhouse:
  15. description: 'Is clickhouse required?'
  16. required: false
  17. default: 'false'
  18. kafka:
  19. description: 'Is kafka required?'
  20. required: false
  21. default: 'false'
  22. chartcuterie:
  23. description: 'Is chartcuterie required?'
  24. required: false
  25. default: 'false'
  26. bigtable:
  27. description: 'Is bigtable required?'
  28. required: false
  29. default: 'false'
  30. python-version:
  31. description: 'python version to install'
  32. required: false
  33. pg-version:
  34. description: 'PostgreSQL version to use'
  35. default: '9.6'
  36. required: false
  37. outputs:
  38. yarn-cache-dir:
  39. description: 'Path to yarn cache'
  40. value: ${{ steps.config.outputs.yarn-cache-dir }}
  41. acceptance-dir:
  42. description: 'Path to acceptance visual snapshot artifacts'
  43. value: ${{ steps.config.outputs.acceptance-dir }}
  44. matrix-instance-number:
  45. description: 'The matrix instance number (starting at 1)'
  46. value: ${{ steps.config.outputs.matrix-instance-number }}
  47. matrix-instance-total:
  48. description: 'Reexport of MATRIX_INSTANCE_TOTAL.'
  49. value: ${{ steps.config.outputs.matrix-instance-total }}
  50. runs:
  51. using: 'composite'
  52. steps:
  53. - name: Setup default environment variables
  54. shell: bash
  55. env:
  56. MATRIX_INSTANCE: ${{ matrix.instance }}
  57. # XXX: We should be using something like len(strategy.matrix.instance) (not possible atm)
  58. # If you have other things like python-version: [foo, bar, baz] then the sharding logic
  59. # isn't right because job-total will be 3x larger and you'd never run 2/3 of the tests.
  60. # MATRIX_INSTANCE_TOTAL: ${{ strategy.job-total }}
  61. run: |
  62. # Only set `MIGRATIONS_TEST_MIGRATE` if it is not already set (or if it's an empty string)
  63. if [ -z $MIGRATIONS_TEST_MIGRATE ]; then
  64. echo "MIGRATIONS_TEST_MIGRATE=0" >> $GITHUB_ENV
  65. fi
  66. echo "PIP_DISABLE_PIP_VERSION_CHECK=on" >> $GITHUB_ENV
  67. echo "PY_COLORS=1" >> "$GITHUB_ENV"
  68. echo "SENTRY_SKIP_BACKEND_VALIDATION=1" >> $GITHUB_ENV
  69. ### node configuration ###
  70. echo "NODE_ENV=development" >> $GITHUB_ENV
  71. echo "NODE_OPTIONS=--max-old-space-size=4096" >> $GITHUB_ENV
  72. ### pytest-sentry configuration ###
  73. echo "PYTEST_SENTRY_DSN=https://6fd5cfea2d4d46b182ad214ac7810508@sentry.io/2423079" >> $GITHUB_ENV
  74. echo "PYTEST_ADDOPTS=--reruns 5" >> $GITHUB_ENV
  75. # Configure a different release version, otherwise it defaults to the
  76. # commit sha which will conflict with our actual prod releases. This is a
  77. # confusing experience because it looks like these are "empty" releases
  78. # because no commits are attached and associates the release with our
  79. # javascript + sentry projects.
  80. echo "SENTRY_RELEASE=ci@$GITHUB_SHA" >> $GITHUB_ENV
  81. # this handles pytest test sharding
  82. if [ "$MATRIX_INSTANCE" ]; then
  83. if ! [ "$MATRIX_INSTANCE_TOTAL" ]; then
  84. echo "MATRIX_INSTANCE_TOTAL is required."
  85. exit 1
  86. fi
  87. echo "TEST_GROUP=$MATRIX_INSTANCE" >> $GITHUB_ENV
  88. echo "TOTAL_TEST_GROUPS=$MATRIX_INSTANCE_TOTAL" >> $GITHUB_ENV
  89. fi
  90. # This records failures on master to sentry in order to detect flakey tests, as it's
  91. # expected that people have failing tests on their PRs
  92. [ "$GITHUB_REF" = "refs/heads/master" ] && echo "PYTEST_SENTRY_ALWAYS_REPORT=1" >> $GITHUB_ENV || true
  93. - name: Setup python
  94. uses: actions/setup-python@v4
  95. with:
  96. python-version: ${{ inputs.python-version }}
  97. cache: pip
  98. cache-dependency-path: ${{ inputs.workdir }}/requirements-dev-frozen.txt
  99. - name: Install system dependencies
  100. shell: bash
  101. run: |
  102. sudo apt-get update
  103. sudo apt-get install -y --no-install-recommends \
  104. libxmlsec1-dev \
  105. libmaxminddb-dev
  106. - name: Set up outputs
  107. id: config
  108. env:
  109. MATRIX_INSTANCE: ${{ matrix.instance }}
  110. shell: bash
  111. run: |
  112. echo "::set-output name=yarn-cache-dir::$(yarn cache dir)"
  113. echo "::set-output name=matrix-instance-number::$(($MATRIX_INSTANCE+1))"
  114. echo "::set-output name=matrix-instance-total::$(($MATRIX_INSTANCE_TOTAL))"
  115. echo "::set-output name=acceptance-dir::.artifacts/visual-snapshots/acceptance"
  116. - name: Install python dependencies
  117. shell: bash
  118. env:
  119. # This is necessary when other repositories (e.g. relay) want to take advantage of this workflow
  120. # without needing to fork it. The path needed is the one where setup.py is located
  121. WORKDIR: ${{ inputs.workdir }}
  122. run: |
  123. cd "$WORKDIR"
  124. python setup.py install_egg_info
  125. # We need to install editable otherwise things like check migration will fail.
  126. pip install -r requirements-dev-frozen.txt & \
  127. SENTRY_LIGHT_BUILD=1 pip install --no-deps -e . & \
  128. wait
  129. cd -
  130. - name: Start devservices
  131. shell: bash
  132. env:
  133. NEED_KAFKA: ${{ inputs.kafka }}
  134. NEED_SNUBA: ${{ inputs.snuba }}
  135. NEED_CLICKHOUSE: ${{ inputs.clickhouse }}
  136. NEED_BIGTABLE: ${{ inputs.bigtable }}
  137. NEED_CHARTCUTERIE: ${{ inputs.chartcuterie }}
  138. WORKDIR: ${{ inputs.workdir }}
  139. PG_VERSION: ${{ inputs.pg-version }}
  140. run: |
  141. sentry init
  142. # redis, postgres are needed for almost every code path.
  143. services='redis postgres'
  144. if [ "$NEED_CLICKHOUSE" = "true" ] || [ "$NEED_SNUBA" = "true" ]; then
  145. services="${services} clickhouse"
  146. fi
  147. if [ "$NEED_SNUBA" = "true" ]; then
  148. services="${services} snuba"
  149. fi
  150. if [ "$NEED_BIGTABLE" = "true" ]; then
  151. echo "BIGTABLE_EMULATOR_HOST=127.0.0.1:8086" >> $GITHUB_ENV
  152. services="${services} bigtable"
  153. fi
  154. if [ "$NEED_CHARTCUTERIE" = "true" ]; then
  155. services="${services} chartcuterie"
  156. fi
  157. sentry devservices up $services &
  158. # TODO: Use devservices kafka. See https://github.com/getsentry/sentry/pull/20986#issuecomment-704510570
  159. if [ "$NEED_KAFKA" = "true" ]; then
  160. # This is *not* the production version. Unclear reason as to why this was chosen
  161. # https://github.com/getsentry/ops/blob/c823e62f930ecc6c97bb08898c71e49edc7232f6/cookbooks/getsentry/attributes/default.rb#L631
  162. docker run \
  163. --name sentry_zookeeper \
  164. -d --network host \
  165. -e ZOOKEEPER_CLIENT_PORT=2181 \
  166. confluentinc/cp-zookeeper:4.1.0 \
  167. &
  168. # This is the production version; do not change w/o changing it there as well
  169. # https://github.com/getsentry/ops/blob/c823e62f930ecc6c97bb08898c71e49edc7232f6/cookbooks/getsentry/attributes/default.rb#L643
  170. docker run \
  171. --name sentry_kafka \
  172. -d --network host \
  173. -e KAFKA_ZOOKEEPER_CONNECT=127.0.0.1:2181 \
  174. -e KAFKA_LISTENERS=INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092 \
  175. -e KAFKA_ADVERTISED_LISTENERS=INTERNAL://127.0.0.1:9093,EXTERNAL://127.0.0.1:9092 \
  176. -e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT \
  177. -e KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL \
  178. -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \
  179. confluentinc/cp-kafka:5.1.2 \
  180. &
  181. fi
  182. wait
  183. docker ps -a
  184. ./$WORKDIR/scripts/devservices-healthcheck.sh