Rework CI into multiple dependent jobs (#4823)

.github/workflows/ci.yml

@@ -12,60 +12,266 @@ on:
 permissions:
   contents: read
 
+env:
+  DEFAULT_PYTHON: "3.9"
+  PYUPGRADE_TARGET: "--py39-plus"
+  CLANG_FORMAT_VERSION: "13.0.1"
+
 concurrency:
   # yamllint disable-line rule:line-length
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
-  ci:
-    name: ${{ matrix.name }}
+  common:
+    name: Create common environment
     runs-on: ubuntu-latest
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v3.5.2
+      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
+        uses: actions/setup-python@v4.6.0
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON }}
+      - name: Restore Python virtual environment
+        id: cache-venv
+        uses: actions/cache@v3.3.1
+        with:
+          path: venv
+          # yamllint disable-line rule:line-length
+          key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
+      - name: Create Python virtual environment
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: |
+          python -m venv venv
+          . venv/bin/activate
+          python --version
+          pip install -r requirements.txt -r requirements_optional.txt -r requirements_test.txt
+          pip install -e .
+
+  yamllint:
+    name: yamllint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v3.5.2
+      - name: Run yamllint
+        uses: frenck/action-yamllint@v1.4.0
+
+  black:
+    name: Check black
+    runs-on: ubuntu-latest
+    needs:
+      - common
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v3.5.2
+      - name: Restore Python virtual environment
+        uses: actions/cache/restore@v3.3.1
+        with:
+          path: venv
+          # yamllint disable-line rule:line-length
+          key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
+      - name: Run black
+        run: |
+          . venv/bin/activate
+          black --verbose esphome tests
+      - name: Suggested changes
+        run: script/ci-suggest-changes
+        if: always()
+
+  flake8:
+    name: Check flake8
+    runs-on: ubuntu-latest
+    needs:
+      - common
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v3.5.2
+      - name: Restore Python virtual environment
+        uses: actions/cache/restore@v3.3.1
+        with:
+          path: venv
+          # yamllint disable-line rule:line-length
+          key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
+      - name: Run flake8
+        run: |
+          . venv/bin/activate
+          flake8 esphome
+      - name: Suggested changes
+        run: script/ci-suggest-changes
+        if: always()
+
+  pylint:
+    name: Check pylint
+    runs-on: ubuntu-latest
+    needs:
+      - common
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v3.5.2
+      - name: Restore Python virtual environment
+        uses: actions/cache/restore@v3.3.1
+        with:
+          path: venv
+          # yamllint disable-line rule:line-length
+          key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
+      - name: Run pylint
+        run: |
+          . venv/bin/activate
+          pylint -f parseable --persistent=n esphome
+      - name: Suggested changes
+        run: script/ci-suggest-changes
+        if: always()
+
+  pyupgrade:
+    name: Check pyupgrade
+    runs-on: ubuntu-latest
+    needs:
+      - common
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v3.5.2
+      - name: Restore Python virtual environment
+        uses: actions/cache/restore@v3.3.1
+        with:
+          path: venv
+          # yamllint disable-line rule:line-length
+          key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
+      - name: Run pyupgrade
+        run: |
+          . venv/bin/activate
+          pyupgrade ${{ env.PYUPGRADE_TARGET }} `find esphome -name "*.py" -type f`
+      - name: Suggested changes
+        run: script/ci-suggest-changes
+        if: always()
+
+  ci-custom:
+    name: Run script/ci-custom
+    runs-on: ubuntu-latest
+    needs:
+      - common
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v3.5.2
+      - name: Restore Python virtual environment
+        uses: actions/cache/restore@v3.3.1
+        with:
+          path: venv
+          # yamllint disable-line rule:line-length
+          key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
+      - name: Register matcher
+        run: echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
+      - name: Run script/ci-custom
+        run: |
+          . venv/bin/activate
+          script/ci-custom.py
+          script/build_codeowners.py --check
+
+  pytest:
+    name: Run pytest
+    runs-on: ubuntu-latest
+    needs:
+      - common
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v3.5.2
+      - name: Restore Python virtual environment
+        uses: actions/cache/restore@v3.3.1
+        with:
+          path: venv
+          # yamllint disable-line rule:line-length
+          key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
+      - name: Register matcher
+        run: echo "::add-matcher::.github/workflows/matchers/pytest.json"
+      - name: Run pytest
+        run: |
+          . venv/bin/activate
+          pytest -vv --tb=native tests
+
+  clang-format:
+    name: Check clang-format
+    runs-on: ubuntu-latest
+    needs:
+      - common
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v3.5.2
+      - name: Restore Python virtual environment
+        uses: actions/cache/restore@v3.3.1
+        with:
+          path: venv
+          # yamllint disable-line rule:line-length
+          key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
+      - name: Install clang-format
+        run: |
+          . venv/bin/activate
+          pip install clang-format==${{ env.CLANG_FORMAT_VERSION }}
+      - name: Run clang-format
+        run: |
+          . venv/bin/activate
+          script/clang-format -i
+          git diff-index --quiet HEAD --
+      - name: Suggested changes
+        run: script/ci-suggest-changes
+        if: always()
+
+  compile-tests:
+    name: Run YAML test ${{ matrix.file }}
+    runs-on: ubuntu-latest
+    needs:
+      - common
+      - black
+      - ci-custom
+      - clang-format
+      - flake8
+      - pylint
+      - pytest
+      - pyupgrade
+      - yamllint
     strategy:
       fail-fast: false
-      max-parallel: 5
+      max-parallel: 2
+      matrix:
+        file: [1, 2, 3, 3.1, 4, 5, 6, 7]
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v3.5.2
+      - name: Restore Python virtual environment
+        uses: actions/cache/restore@v3.3.1
+        with:
+          path: venv
+          # yamllint disable-line rule:line-length
+          key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
+      - name: Cache platformio
+        uses: actions/cache@v3.3.1
+        with:
+          path: ~/.platformio
+          # yamllint disable-line rule:line-length
+          key: platformio-test${{ matrix.file }}-${{ hashFiles('platformio.ini') }}
+      - name: Run esphome compile tests/test${{ matrix.file }}.yaml
+        run: |
+          . venv/bin/activate
+          esphome compile tests/test${{ matrix.file }}.yaml
+
+  clang-tidy:
+    name: ${{ matrix.name }}
+    runs-on: ubuntu-latest
+    needs:
+      - common
+      - black
+      - ci-custom
+      - clang-format
+      - flake8
+      - pylint
+      - pytest
+      - pyupgrade
+      - yamllint
+    strategy:
+      fail-fast: false
+      max-parallel: 2
       matrix:
         include:
-          - id: ci-custom
-            name: Run script/ci-custom
-          - id: lint-python
-            name: Run script/lint-python
-          - id: test
-            file: tests/test1.yaml
-            name: Test tests/test1.yaml
-            pio_cache_key: test1
-          - id: test
-            file: tests/test2.yaml
-            name: Test tests/test2.yaml
-            pio_cache_key: test2
-          - id: test
-            file: tests/test3.yaml
-            name: Test tests/test3.yaml
-            pio_cache_key: test3
-          - id: test
-            file: tests/test3.1.yaml
-            name: Test tests/test3.1.yaml
-            pio_cache_key: test3.1
-          - id: test
-            file: tests/test4.yaml
-            name: Test tests/test4.yaml
-            pio_cache_key: test4
-          - id: test
-            file: tests/test5.yaml
-            name: Test tests/test5.yaml
-            pio_cache_key: test5
-          - id: test
-            file: tests/test6.yaml
-            name: Test tests/test6.yaml
-            pio_cache_key: test6
-          - id: test
-            file: tests/test7.yaml
-            name: Test tests/test7.yaml
-            pio_cache_key: test7
-          - id: pytest
-            name: Run pytest
-          - id: clang-format
-            name: Run script/clang-format
           - id: clang-tidy
             name: Run script/clang-tidy for ESP8266
             options: --environment esp8266-arduino-tidy --grep USE_ESP8266
@@ -90,119 +296,65 @@ jobs:
             name: Run script/clang-tidy for ESP32 IDF
             options: --environment esp32-idf-tidy --grep USE_ESP_IDF
             pio_cache_key: tidyesp32-idf
-          - id: yamllint
-            name: Run yamllint
 
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        id: python
+      - name: Check out code from GitHub
+        uses: actions/checkout@v3.5.2
+      - name: Restore Python virtual environment
+        uses: actions/cache/restore@v3.3.1
         with:
-          python-version: "3.9"
-
-      - name: Cache virtualenv
-        uses: actions/cache@v3
-        with:
-          path: .venv
+          path: venv
           # yamllint disable-line rule:line-length
-          key: venv-${{ steps.python.outputs.python-version }}-${{ hashFiles('requirements*.txt') }}
-          restore-keys: |
-            venv-${{ steps.python.outputs.python-version }}-
-
-      - name: Set up virtualenv
-        # yamllint disable rule:line-length
-        run: |
-          python -m venv .venv
-          source .venv/bin/activate
-          pip install -U pip
-          pip install -r requirements.txt -r requirements_optional.txt -r requirements_test.txt
-          pip install -e .
-          echo "$GITHUB_WORKSPACE/.venv/bin" >> $GITHUB_PATH
-          echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> $GITHUB_ENV
-        # yamllint enable rule:line-length
-
+          key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
         # Use per check platformio cache because checks use different parts
       - name: Cache platformio
-        uses: actions/cache@v3
+        uses: actions/cache@v3.3.1
         with:
           path: ~/.platformio
           # yamllint disable-line rule:line-length
           key: platformio-${{ matrix.pio_cache_key }}-${{ hashFiles('platformio.ini') }}
-        if: matrix.id == 'test' || matrix.id == 'clang-tidy'
 
-      - name: Install clang tools
-        run: |
-          sudo apt-get install \
-              clang-format-13 \
-              clang-tidy-11
-        if: matrix.id == 'clang-tidy' || matrix.id == 'clang-format'
+      - name: Install clang-tidy
+        run: sudo apt-get install clang-tidy-11
 
       - name: Register problem matchers
         run: |
-          echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
-          echo "::add-matcher::.github/workflows/matchers/lint-python.json"
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-          echo "::add-matcher::.github/workflows/matchers/pytest.json"
           echo "::add-matcher::.github/workflows/matchers/gcc.json"
           echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
 
-      - name: Lint Custom
-        run: |
-          script/ci-custom.py
-          script/build_codeowners.py --check
-        if: matrix.id == 'ci-custom'
-
-      - name: Lint Python
-        run: script/lint-python -a
-        if: matrix.id == 'lint-python'
-
-      - run: esphome compile ${{ matrix.file }}
-        if: matrix.id == 'test'
-        env:
-          # Also cache libdeps, store them in a ~/.platformio subfolder
-          PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps
-
-      - name: Run pytest
-        run: |
-          pytest -vv --tb=native tests
-        if: matrix.id == 'pytest'
-
-      # Also run git-diff-index so that the step is marked as failed on
-      # formatting errors, since clang-format doesn't do anything but
-      # change files if -i is passed.
-      - name: Run clang-format
-        run: |
-          script/clang-format -i
-          git diff-index --quiet HEAD --
-        if: matrix.id == 'clang-format'
-
       - name: Run clang-tidy
         run: |
+          . venv/bin/activate
           script/clang-tidy --all-headers --fix ${{ matrix.options }}
-        if: matrix.id == 'clang-tidy'
         env:
           # Also cache libdeps, store them in a ~/.platformio subfolder
           PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps
 
-      - name: Run yamllint
-        if: matrix.id == 'yamllint'
-        uses: frenck/action-yamllint@v1.4.0
-
       - name: Suggested changes
         run: script/ci-suggest-changes
         # yamllint disable-line rule:line-length
-        if: always() && (matrix.id == 'clang-tidy' || matrix.id == 'clang-format' || matrix.id == 'lint-python')
+        if: always()
 
   ci-status:
     name: CI Status
     runs-on: ubuntu-latest
-    needs: [ci]
+    needs:
+      - common
+      - black
+      - ci-custom
+      - clang-format
+      - flake8
+      - pylint
+      - pytest
+      - pyupgrade
+      - yamllint
+      - compile-tests
+      - clang-tidy
     if: always()
     steps:
-      - name: Successful deploy
+      - name: Success
         if: ${{ !(contains(needs.*.result, 'failure')) }}
         run: exit 0
-      - name: Failing deploy
+      - name: Failure
         if: ${{ contains(needs.*.result, 'failure') }}
         run: exit 1