CI: general restructure (#7407)

Co-authored-by: Greg Korba <wirone@gmail.com>

.github/composite-actions/install-composer-deps/action.yml

@@ -0,0 +1,23 @@
+name: "Install composer deps"
+
+inputs:
+  flags:
+    description: 'Composer flags'
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install dependencies
+      uses: nick-invision/retry@v2
+      with:
+        timeout_minutes: 5
+        max_attempts: 5
+        retry_wait_seconds: 30
+        # `--no-scripts` to avoid side-effects (e.g. installing dev-tools for all jobs on CI level),
+        # all executed scripts should be explicit and run only when needed.
+        command: composer update --optimize-autoloader --no-interaction --no-progress --no-scripts ${{ inputs.flags }}
+
+    - name: Show versions of packages
+      shell: bash
+      run: composer info -D

.github/composite-actions/setup-php-with-composer-deps/action.yml

@@ -0,0 +1,57 @@
+name: "Setup PHP with Composer deps"
+
+inputs:
+  os:
+    description: 'OS version'
+    required: true
+  php:
+    description: 'PHP version'
+    required: true
+  php-coverage:
+    description: 'Enable coverage driver'
+    required: false
+    default: 'no'
+  tools:
+    description: 'tools'
+    required: false
+    default: 'none'
+  composer-flags:
+    description: 'Composer flags'
+    required: false
+  composer-flex-with-symfony-version:
+    description: 'Sf version to determine with Flex'
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    - name: Resolve PHP tools flag
+      uses: actions/github-script@v6
+      id: setup-php-resolve-tools
+      with:
+        script: 'return "${{ inputs.composer-flex-with-symfony-version }}" ? "flex" : "none"'
+        result-encoding: string
+
+    - name: Resolve PHP coverage flag
+      uses: actions/github-script@v6
+      id: setup-php-resolve-coverage
+      with:
+        script: 'return "${{ inputs.php-coverage }}" == "yes" ? "pcov" : "none"'
+        result-encoding: string
+
+    - name: Setup PHP
+      uses: ./.github/composite-actions/setup-php
+      with:
+        version: ${{ inputs.php }}
+        coverage: ${{ steps.setup-php-resolve-coverage.outputs.result }}
+        tools: ${{ steps.setup-php-resolve-tools.outputs.result }}, ${{ inputs.tools }}
+
+    - name: Configure Symfony Flex
+      if: inputs.composer-flex-with-symfony-version
+      shell: bash
+      run: composer config extra.symfony.require ${{ inputs.composer-flex-with-symfony-version }}
+
+    - name: Install Composer deps
+      uses: ./.github/composite-actions/install-composer-deps
+      with:
+        flags: ${{ inputs.composer-flags }}

.github/composite-actions/setup-php/action.yml

@@ -0,0 +1,25 @@
+name: "Setup PHP"
+
+inputs:
+  version:
+    description: 'PHP version'
+    required: true
+  coverage:
+    description: 'coverage mode'
+    required: false
+    default: 'none'
+  tools:
+    description: 'tools'
+    required: false
+    default: 'none'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup PHP
+      uses: shivammathur/setup-php@v2
+      with:
+        php-version: ${{ inputs.version }}
+        coverage: ${{ inputs.coverage }}
+        tools: composer, ${{ inputs.tools }}
+        extensions: none, curl, dom, json, mbstring, opcache, openssl, simplexml, tokenizer, xml, xmlwriter, zip

.github/workflows/ci.yml

@@ -11,6 +11,10 @@ concurrency:
   group: ci-${{ github.head_ref || github.run_id }} # will be canceled on subsequent pushes in pull requests but not branches
   cancel-in-progress: true
 
+env:
+  PHP_MAX: '8.2'
+  PHP_MIN: '7.4'
+
 jobs:
   tests:
     strategy:
@@ -21,7 +25,6 @@ jobs:
             php-version: '7.4'
             job-description: 'Fixer with lowest deps'
             run-fixer: 'yes'
-            run-phpdoc-to-native-type: 'yes' # should be checked on the lowest supported PHP version
             composer-flags: '--prefer-lowest' # should be checked on the lowest supported PHP version
             execute-flex-with-symfony-version: '^5' # explicit check for Symfony 5.x compatibility
 
@@ -67,31 +70,21 @@ jobs:
 
           - operating-system: 'ubuntu-20.04'
             php-version: '8.2'
-            job-description: 'tests with migration rules'
-            run-tests: 'yes'
-            run-migration-rules: 'yes' # should be checked on the highest supported PHP version
-
-          - operating-system: 'ubuntu-20.04'
-            php-version: '8.2'
-            job-description: 'auto-review'
+            job-description: 'tests'
             run-tests: 'yes'
-            phpunit-flags: '--group auto-review'
 
           - operating-system: 'ubuntu-20.04'
             php-version: '8.2'
-            job-description: 'tests'
+            job-description: 'tests with migration rules'
             run-tests: 'yes'
+            run-migration-rules: 'yes' # should be checked on the highest supported PHP version
 
           - operating-system: 'ubuntu-20.04'
             php-version: '8.2'
             job-description: 'code coverage'
+            run-tests: 'yes'
             collect-code-coverage: 'yes'
 
-          - operating-system: 'ubuntu-20.04'
-            php-version: '8.2'
-            job-description: 'deployment check'
-            execute-deployment: 'yes'
-
           - operating-system: 'windows-latest'
             php-version: '8.2'
             job-description: 'Fixer on Windows'
@@ -138,62 +131,22 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
 
-      - name: Resolve PHP coverage flag
-        uses: actions/github-script@v6
-        id: setup-php-resolve-coverage
-        with:
-          script: 'return "${{ matrix.code-coverage }}" == "yes" ? "pcov" : "none"'
-          result-encoding: string
-
-      - name: Resolve PHP tools flag
-        uses: actions/github-script@v6
-        id: setup-php-resolve-tools
-        with:
-          script: 'return "${{ matrix.execute-flex-with-symfony-version }}" ? "flex" : "none"'
-          result-encoding: string
-
-      - name: Setup PHP
-        uses: shivammathur/setup-php@v2
-        with:
-          php-version: ${{ matrix.php-version }}
-          extensions: none, curl, dom, json, mbstring, opcache, openssl, simplexml, tokenizer, xml, xmlwriter, zip
-          coverage: ${{ steps.setup-php-resolve-coverage.outputs.result }}
-          tools: composer, ${{ steps.setup-php-resolve-tools.outputs.result }}
-
-      - name: Get Composer cache directory
-        id: composer-cache
-        shell: bash
-        run: echo "dir=$(composer config cache-dir)" >> $GITHUB_OUTPUT
-
-      - name: Cache dependencies
-        uses: actions/cache@v3
+      - name: Setup PHP with Composer deps
+        uses: ./.github/composite-actions/setup-php-with-composer-deps
         with:
-          path: ${{ steps.composer-cache.outputs.dir }}
-          key: composer-${{ runner.os }}-${{ matrix.php-version }}-${{ hashFiles('**/composer.*') }}-${{ matrix.composer-flags }}
-          restore-keys: |
-            composer-${{ runner.os }}-${{ matrix.php-version }}-${{ hashFiles('**/composer.*') }}-
-            composer-${{ runner.os }}-${{ matrix.php-version }}-
-            composer-${{ runner.os }}-
-            composer-
-
-      - name: Configure Symfony Flex
-        if: matrix.execute-flex-with-symfony-version
-        run: composer config extra.symfony.require ${{ matrix.execute-flex-with-symfony-version }}
-
-      - name: Install dependencies
-        uses: nick-invision/retry@v2
-        with:
-          timeout_minutes: 5
-          max_attempts: 5
-          retry_wait_seconds: 30
-          command: |
-            composer update --optimize-autoloader --no-interaction --no-progress --no-scripts ${{ matrix.composer-flags }} # --no-scripts to avoid installing dev-tools for all jobs on CI level
-            composer info -D
-
-      # execute migration rules before running tests and self-fixing,
-      # so that we know that our codebase is future-ready
+          os: ${{ runner.os }}
+          php: ${{ matrix.php-version }}
+          php-coverage: ${{ matrix.collect-code-coverage }}
+          composer-flags: ${{ matrix.composer-flags }}
+          composer-flex-with-symfony-version: ${{ matrix.execute-flex-with-symfony-version }}
+
+      # Execute migration rules before running tests and self-fixing,
+      # so we know that our codebase is future-ready.
+      # Should be checked on the highest supported PHP version.
       - name: Run PHP CS Fixer with migration rules
-        if: matrix.run-migration-rules == 'yes'
+        if: env.PHP_MAX == matrix.php-version && matrix.run-migration-rules == 'yes'
+        env:
+          PHP_CS_FIXER_FUTURE_MODE: 1
         run: php php-cs-fixer fix --config .php-cs-fixer.php-highest.php -q
 
       - name: Disable time limit for tests when collecting coverage
@@ -205,24 +158,20 @@ jobs:
         env:
           PHP_CS_FIXER_IGNORE_ENV: ${{ matrix.PHP_CS_FIXER_IGNORE_ENV }}
           FAST_LINT_TEST_CASES: ${{ matrix.FAST_LINT_TEST_CASES }}
-        run: vendor/bin/paraunit run ${{ matrix.phpunit-flags || '--exclude-group auto-review' }}
+        run: vendor/bin/paraunit run --testsuite unit,integration
 
-      - name: Collect code coverage
-        if: matrix.collect-code-coverage == 'yes'
+      - name: Run tests and collect code coverage
+        if: matrix.run-tests == 'yes' && matrix.collect-code-coverage == 'yes'
         env:
           FAST_LINT_TEST_CASES: 1
-        run: vendor/bin/paraunit coverage --testsuite coverage --exclude-group covers-nothing --clover build/logs/clover.xml
+        run: vendor/bin/paraunit coverage --testsuite unit --exclude-group covers-nothing --clover build/logs/clover.xml
 
       - name: Upload coverage results to Coveralls
-        if: matrix.collect-code-coverage == 'yes'
+        if: matrix.run-tests == 'yes' && matrix.collect-code-coverage == 'yes'
         env:
           COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: php vendor/bin/php-coveralls --verbose
 
-      - name: Run PHP CS Fixer with PHPDoc to type rules
-        if: matrix.run-phpdoc-to-native-type == 'yes'
-        run: php php-cs-fixer check --diff -v --config .php-cs-fixer.php-lowest.php
-
       - name: Run PHP CS Fixer
         if: matrix.run-fixer == 'yes'
         env:
@@ -230,8 +179,41 @@ jobs:
           PHP_CS_FIXER_FUTURE_MODE: 1
         run: php php-cs-fixer check --diff -v
 
-      - name: Execute deployment checks
-        if: matrix.execute-deployment == 'yes'
-        run: |
-          ./dev-tools/build.sh
-          PHP_CS_FIXER_TEST_ALLOW_SKIPPING_SMOKE_TESTS=0 vendor/bin/phpunit tests/Smoke/
+      # Should be checked on the lowest supported PHP version.
+      # If any type can be converted from PHPDoc to native type on lowest supported PHP, we should commit such change.
+      - name: Run PHP CS Fixer with PHPDoc to type rules
+        if: env.PHP_MIN == matrix.php-version && matrix.run-fixer == 'yes'
+        env:
+          PHP_CS_FIXER_FUTURE_MODE: 1
+        run: php php-cs-fixer check --diff -v --config .php-cs-fixer.php-lowest.php
+
+  deployment:
+    needs: tests
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - operating-system: 'ubuntu-20.04'
+            php-version: '8.2'
+
+    name: Deployment checks
+
+    runs-on: ${{ matrix.operating-system }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Setup PHP with Composer deps
+        uses: ./.github/composite-actions/setup-php-with-composer-deps
+        with:
+          os: ${{ runner.os }}
+          php: ${{ matrix.php-version }}
+
+      - name: Build phar
+        run: ./dev-tools/build.sh
+
+      - name: Run smoke tests
+        env:
+          FAST_LINT_TEST_CASES: 1
+        run: vendor/bin/phpunit --testsuite smoke

.github/workflows/sca.yml

@@ -28,43 +28,14 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
 
-      - name: Setup PHP
-        uses: shivammathur/setup-php@v2
+      - name: Setup PHP with Composer deps
+        uses: ./.github/composite-actions/setup-php-with-composer-deps
         with:
-          php-version: ${{ matrix.php-version }}
-          extensions: none, curl, dom, json, mbstring, opcache, simplexml, tokenizer, xml, xmlwriter, zip
-          coverage: none # without this Xdebug will be enabled
+          os: ${{ runner.os }}
+          php: ${{ matrix.php-version }}
           tools: cs2pr
 
-      - name: Get Composer cache directory
-        id: composer-cache
-        run: echo "dir=$(composer config cache-dir)" >> $GITHUB_OUTPUT
-
-      - name: Cache dependencies
-        uses: actions/cache@v3
-        with:
-          path: ${{ steps.composer-cache.outputs.dir }}
-          key: composer-${{ runner.os }}-${{ matrix.php-version }}-${{ hashFiles('**/composer.*') }}-${{ matrix.composer-flags }}
-          restore-keys: |
-            composer-${{ runner.os }}-${{ matrix.php-version }}-${{ hashFiles('**/composer.*') }}-
-            composer-${{ runner.os }}-${{ matrix.php-version }}-
-            composer-${{ runner.os }}-
-            composer-
-
-      - name: Install dependencies
-        uses: nick-invision/retry@v2
-        with:
-          timeout_minutes: 5
-          max_attempts: 5
-          retry_wait_seconds: 30
-          command: composer update --optimize-autoloader --no-interaction --no-progress ${{ matrix.composer-flags }}
-
-      - name: Report versions
-        run: composer info -D
-
       ## We want to have a lock-file used on PR level, so contributors are not bothered by SCA complains unrelated to their changes,
       ## and same time we want to be aware that we are complying with bleeding edge of SCA tools as maintainers observing the push hook.
       - name: Unlock dev-tools
@@ -79,6 +50,9 @@ jobs:
           retry_wait_seconds: 30
           command: ./dev-tools/install.sh
 
+      - name: Run AutoReview
+        run: vendor/bin/paraunit run --testsuite auto-review
+
       - name: Check - file permissions
         run: ./dev-tools/check_file_permissions.sh
 

composer.json

@@ -81,6 +81,10 @@
         "post-autoload-dump": [
             "@install-tools"
         ],
+        "auto-review": [
+            "Composer\\Config::disableProcessTimeout",
+            "paraunit run --testsuite auto-review"
+        ],
         "cs:check": "@php php-cs-fixer check --diff",
         "cs:fix": "@php php-cs-fixer fix",
         "cs:fix:parallel": "echo '🔍 Will run in batches of 50 files.'; if [[ -f .php-cs-fixer.php ]]; then FIXER_CONFIG=.php-cs-fixer.php; else FIXER_CONFIG=.php-cs-fixer.dist.php; fi; php php-cs-fixer list-files --config=$FIXER_CONFIG | xargs -n 50 -P 8 php php-cs-fixer fix --config=$FIXER_CONFIG --path-mode intersection 2> /dev/null",
@@ -98,7 +102,7 @@
             "Composer\\Config::disableProcessTimeout",
             "@install-tools --quiet",
             "@self-check",
-            "@sa",
+            "@static-analysis",
             "@test"
         ],
         "require-checker": "@php dev-tools/vendor/bin/composer-require-checker check composer.json --config-file .composer-require-checker.json",
@@ -107,7 +111,8 @@
             "./dev-tools/check_file_permissions.sh",
             "./dev-tools/check_trailing_spaces.sh",
             "@normalize",
-            "@require-checker"
+            "@require-checker",
+            "@auto-review"
         ],
         "static-analysis": [
             "@cs:check",
@@ -116,15 +121,28 @@
         ],
         "test": "@test:all",
         "test:all": [
-            "Composer\\Config::disableProcessTimeout",
-            "paraunit run --testsuite all"
+            "@test:unit",
+            "@test:integration"
         ],
         "test:coverage": [
             "Composer\\Config::disableProcessTimeout",
-            "paraunit run --testsuite coverage"
+            "paraunit coverage --testsuite unit --exclude-group covers-nothing"
+        ],
+        "test:integration": [
+            "Composer\\Config::disableProcessTimeout",
+            "paraunit run --testsuite integration"
+        ],
+        "test:smoke": [
+            "Composer\\Config::disableProcessTimeout",
+            "paraunit run --testsuite smoke"
+        ],
+        "test:unit": [
+            "Composer\\Config::disableProcessTimeout",
+            "paraunit run --testsuite unit"
         ]
     },
     "scripts-descriptions": {
+        "auto-review": "Execute Auto-review",
         "cs:check": "Check coding standards",
         "cs:fix": "Fix coding standards",
         "cs:fix:parallel": "Fix coding standards in naive parallel mode (using xargs)",
@@ -142,7 +160,10 @@
         "self-check": "Run set of self-checks ensuring repository's validity",
         "static-analysis": "Run static analysis",
         "test": "Alias for 'test:all'",
-        "test:all": "Run all tests",
-        "test:coverage": "Run tool-related tests"
+        "test:all": "Run Unit and Integration tests (but *NOT* Smoke tests)",
+        "test:coverage": "Run tests that provide code coverage",
+        "test:integration": "Run Integration tests",
+        "test:smoke": "Run Smoke tests",
+        "test:unit": "Run Unit tests"
     }
 }

dev-tools/build.sh

@@ -6,7 +6,7 @@ composer config platform.php 7.4
 
 # install package deps without dev-deps / remove already installed dev-deps
 # box can ignore dev-deps, but dev-deps, when installed, may lower version of prod-deps
-composer update --no-interaction --no-progress --optimize-autoloader --no-scripts --no-dev
+composer update --optimize-autoloader --no-interaction --no-progress --no-scripts --no-dev
 composer info -D | sort
 
 composer show -d dev-tools humbug/box -q || composer update -d dev-tools --no-interaction --no-progress
@@ -16,4 +16,4 @@ dev-tools/vendor/bin/box compile
 
 # revert changes to composer
 git checkout composer.json
-composer update --no-interaction --no-progress -q
+composer update --optimize-autoloader --no-interaction --no-progress --no-scripts -q

doc/usage.rst

@@ -25,7 +25,7 @@ With some magic of tools provided by your OS, you can also fix files in parallel
 
 .. code-block:: console
 
-    php php-cs-fixer.phar list-files --config=.php-cs-fixer.dist.php | xargs -n 10 -P 8 php php-cs-fixer.phar fix --config=.php-cs-fixer.dist.php --path-mode intersection -v
+    php php-cs-fixer.phar list-files --config=.php-cs-fixer.dist.php | xargs -n 50 -P 8 php php-cs-fixer.phar fix --config=.php-cs-fixer.dist.php --path-mode intersection -v
 
 You can limit process to given file or files in a given directory and its subdirectories:
 
@@ -169,7 +169,7 @@ Note: You need to pass the config to the ``fix`` command, in order to make it wo
 
 .. code-block:: console
 
-    php php-cs-fixer.phar list-files --config=.php-cs-fixer.dist.php | xargs -n 10 -P 8 php php-cs-fixer.phar fix --config=.php-cs-fixer.dist.php --path-mode intersection -v
+    php php-cs-fixer.phar list-files --config=.php-cs-fixer.dist.php | xargs -n 50 -P 8 php php-cs-fixer.phar fix --config=.php-cs-fixer.dist.php --path-mode intersection -v
 
 * ``-n`` defines how many files a single subprocess process
 * ``-P`` defines how many subprocesses the shell is allowed to spawn for parallel processing (usually similar to the number of CPUs your system has)

phpunit.xml.dist

@@ -18,13 +18,20 @@
     timeoutForLargeTests="30"
 >
     <testsuites>
-        <testsuite name="all">
-            <directory>./tests</directory>
+        <testsuite name="unit">
+            <directory>./tests/</directory>
+            <exclude>./tests/IntegrationTest.php</exclude>
+            <exclude>./tests/AutoReview/</exclude>
+            <exclude>./tests/Smoke/</exclude>
         </testsuite>
-        <testsuite name="coverage">
-            <directory>./tests</directory>
-            <exclude>./tests/AutoReview</exclude>
-            <exclude>./tests/Smoke</exclude>
+        <testsuite name="integration">
+            <file>./tests/IntegrationTest.php</file>
+        </testsuite>
+        <testsuite name="smoke">
+            <directory>./tests/Smoke/</directory>
+        </testsuite>
+        <testsuite name="auto-review">
+            <directory>./tests/AutoReview/</directory>
         </testsuite>
     </testsuites>
 
@@ -42,6 +49,5 @@
         <ini name="zend.enable_gc" value="0"/>
         <ini name="memory_limit" value="10G"/>
         <env name="FAST_LINT_TEST_CASES" value="0"/>
-        <env name="PHP_CS_FIXER_TEST_ALLOW_SKIPPING_SMOKE_TESTS" value="1"/>
     </php>
 </phpunit>

tests/AutoReview/CiConfigurationTest.php

@@ -32,6 +32,17 @@ use Symfony\Component\Yaml\Yaml;
  */
 final class CiConfigurationTest extends TestCase
 {
+    public function testThatPhpVersionEnvsAreSetProperly(): void
+    {
+        self::assertSame(
+            [
+                'PHP_MAX' => $this->getMaxPhpVersionFromEntryFile(),
+                'PHP_MIN' => $this->getMinPhpVersionFromEntryFile(),
+            ],
+            $this->getGitHubCiEnvs(),
+        );
+    }
+
     public function testTestJobsRunOnEachPhp(): void
     {
         $supportedVersions = [];
@@ -154,9 +165,7 @@ final class CiConfigurationTest extends TestCase
      */
     private function getAllPhpVersionsUsedByCiForDeployments(): array
     {
-        $jobs = array_filter($this->getGitHubJobs(), static fn (array $job): bool => isset($job['execute-deployment']) && 'yes' === $job['execute-deployment']);
-
-        return array_map(static fn ($job): string => \is_string($job['php-version']) ? $job['php-version'] : sprintf('%.1f', $job['php-version']), $jobs);
+        return array_map(static fn ($job): string => \is_string($job['php-version']) ? $job['php-version'] : sprintf('%.1f', $job['php-version']), $this->getGitHubDeploymentJobs());
     }
 
     /**
@@ -213,14 +222,24 @@ final class CiConfigurationTest extends TestCase
         return $this->convertPhpVerIdToNiceVer($phpVerId);
     }
 
+    /**
+     * @return array<string, string>
+     */
+    private function getGitHubCiEnvs(): array
+    {
+        $yaml = Yaml::parse(file_get_contents(__DIR__.'/../../.github/workflows/ci.yml'));
+
+        return $yaml['env'];
+    }
+
     /**
      * @return list<array<string, scalar>>
      */
-    private function getGitHubJobs(): array
+    private function getGitHubDeploymentJobs(): array
     {
         $yaml = Yaml::parse(file_get_contents(__DIR__.'/../../.github/workflows/ci.yml'));
 
-        return $yaml['jobs']['tests']['strategy']['matrix']['include'];
+        return $yaml['jobs']['deployment']['strategy']['matrix']['include'];
     }
 
     /**