Separate CI Job to run Pytest collection check (#29923)

Before we attempt to run tests in parallel, we quickly check once
if Pytest collection works. This is in order to avoid costly
parallel test execution if that makes no sense to initialize all
the parallel machines. This check used to be done in "Wait for
CI Inages" step, but running it there has the undesireable
side effect that it is not obvious that it's the collection
that fails, also it prevents other jobs (for example
static checks and docs building) from running. This means that
the contributor does not get all the feedback that could be
given immediately.

This PR separates the collection into separate job and only
makes "test" jobs depend on it - all the other jobs that need
CI image depend on "wait for CI image" one and should continue
running even if pytest collection fails.

CI diagrams are also updated to reflect a bit better optionality
and parallelism of the CI jobs.

Author: potiuk

.github/workflows/ci.yml

@@ -540,8 +540,6 @@ jobs:
         env:
           PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
           DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
-      - name: "Tests Pytest collection: ${{matrix.python-version}}"
-        run: breeze shell "python /opt/airflow/scripts/in_container/test_pytest_collection.py"
       - name: "Fix ownership"
         run: breeze ci fix-ownership
         if: always()
@@ -834,13 +832,43 @@ jobs:
       - name: "Post Helm Tests"
         uses: ./.github/actions/post_tests
 
+  test-pytest-collection:
+    timeout-minutes: 5
+    name: "Test Pytest collection"
+    runs-on: "${{needs.build-info.outputs.runs-on}}"
+    needs: [build-info, wait-for-ci-images]
+    if: needs.build-info.outputs.image-build == 'true'
+    env:
+      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
+      BACKEND: sqlite
+      PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
+    steps:
+      - name: Cleanup repo
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+      - name: "Install Breeze"
+        uses: ./.github/actions/breeze
+      - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
+        id: wait-for-images
+        run: breeze ci-image pull --wait-for-image --tag-as-latest
+        env:
+          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+      - name: "Tests Pytest collection"
+        run: breeze shell "python /opt/airflow/scripts/in_container/test_pytest_collection.py"
+      - name: "Fix ownership"
+        run: breeze ci fix-ownership
+        if: always()
+
   tests-postgres:
     timeout-minutes: 130
     name: >
       Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}:
       ${{needs.build-info.outputs.test-types}}
     runs-on: "${{needs.build-info.outputs.runs-on}}"
-    needs: [build-info, wait-for-ci-images]
+    needs: [build-info, test-pytest-collection]
     strategy:
       matrix:
         python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
@@ -884,7 +912,7 @@ jobs:
     name: >
       MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}}
     runs-on: "${{needs.build-info.outputs.runs-on}}"
-    needs: [build-info, wait-for-ci-images]
+    needs: [build-info, test-pytest-collection]
     strategy:
       matrix:
         python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
@@ -928,7 +956,7 @@ jobs:
     name: >
       MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}}
     runs-on: "${{needs.build-info.outputs.runs-on}}"
-    needs: [build-info, wait-for-ci-images]
+    needs: [build-info, test-pytest-collection]
     strategy:
       matrix:
         python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
@@ -970,7 +998,7 @@ jobs:
     name: >
       Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}}
     runs-on: "${{needs.build-info.outputs.runs-on}}"
-    needs: [build-info, wait-for-ci-images]
+    needs: [build-info, test-pytest-collection]
     strategy:
       matrix:
         python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
@@ -1011,7 +1039,7 @@ jobs:
     timeout-minutes: 130
     name: Integration Tests Postgres
     runs-on: "${{needs.build-info.outputs.runs-on}}"
-    needs: [build-info, wait-for-ci-images]
+    needs: [build-info, test-pytest-collection]
     env:
       RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
       TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
@@ -1071,7 +1099,7 @@ jobs:
     timeout-minutes: 130
     name: Integration Tests MySQL
     runs-on: "${{needs.build-info.outputs.runs-on}}"
-    needs: [build-info, wait-for-ci-images]
+    needs: [build-info, test-pytest-collection]
     env:
       RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
       TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
@@ -1112,7 +1140,7 @@ jobs:
     name: "Quarantined tests"
     runs-on: "${{needs.build-info.outputs.runs-on}}"
     continue-on-error: true
-    needs: [build-info, wait-for-ci-images]
+    needs: [build-info, test-pytest-collection]
     env:
       RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
       TEST_TYPES: "Quarantined"

CI.rst

@@ -382,6 +382,8 @@ This workflow is a regular workflow that performs all checks of Airflow code.
 +---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
 | Build docs                      | Builds documentation                                     | Yes      | Yes      | Yes       | Yes               |
 +---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
+| Test Pytest collection          | Tests if pytest collection works                         | Yes      | Yes      | Yes       | Yes               |
++---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
 | Tests                           | Run the Pytest unit tests (Backend/Python matrix)        | Yes      | Yes      | Yes       | Yes (8)           |
 +---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
 | Integration tests               | Runs integration tests (Postgres/Mysql)                  | Yes      | Yes      | Yes       | Yes (9)           |