From cf703f6ac4b1f6b49e16aaacaa51debd97808969 Mon Sep 17 00:00:00 2001 From: Otto Winter Date: Tue, 14 Jul 2020 14:34:44 +0200 Subject: [PATCH] GH Actions Update (#1134) --- .github/workflows/ci.yml | 196 ++++++++++++++ .github/workflows/matchers/ci-custom.json | 16 ++ .github/workflows/matchers/clang-tidy.json | 17 ++ .github/workflows/matchers/gcc.json | 18 ++ .github/workflows/matchers/lint-python.json | 28 ++ .github/workflows/matchers/python.json | 18 ++ .github/workflows/pull.yaml | 59 ----- .github/workflows/release-dev.yaml | 124 --------- .github/workflows/release-dev.yml | 226 ++++++++++++++++ .github/workflows/release-version.yaml | 180 ------------- .github/workflows/release.yml | 276 ++++++++++++++++++++ docker/Dockerfile.lint | 6 +- script/ci-custom.py | 35 ++- script/clang-tidy | 29 +- script/helpers.py | 6 +- 15 files changed, 846 insertions(+), 388 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/matchers/ci-custom.json create mode 100644 .github/workflows/matchers/clang-tidy.json create mode 100644 .github/workflows/matchers/gcc.json create mode 100644 .github/workflows/matchers/lint-python.json create mode 100644 .github/workflows/matchers/python.json delete mode 100644 .github/workflows/pull.yaml delete mode 100644 .github/workflows/release-dev.yaml create mode 100644 .github/workflows/release-dev.yml delete mode 100644 .github/workflows/release-version.yaml create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..9d03705e84 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,196 @@ +# THESE JOBS ARE COPIED IN release.yml and release-dev.yml +# PLEASE ALSO UPDATE THOSE FILES WHEN CHANGING LINES HERE +name: CI + +on: + push: + # On dev branch release-dev already performs CI checks + # On other branches the `pull_request` trigger will be used + branches: [beta, master] + + pull_request: + # Only trigger on certain events (not when comments are added) + types: [opened, reopened, synchronize] + # Only run when PR is against dev branch (all PRs should be against dev branch) + # Helps prevent accidentally merging PRs against master branch + branches: [dev] + +jobs: + # A fast overview job that checks only changed files + overview: + runs-on: ubuntu-latest + container: esphome/esphome-lint:dev + steps: + # Also fetch history and dev branch so that we can check which files changed + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch dev branch + run: git fetch origin dev + + # Cache the .pio directory with (primarily) library dependencies + - name: Cache .pio lib_deps + uses: actions/cache@v1 + with: + path: .pio + key: lint-cpp-pio-${{ hashFiles('platformio.ini') }} + restore-keys: | + lint-cpp-pio- + - name: Set up python environment + run: script/setup + # Set up the pio project so that the cpp checks know how files are compiled + # (build flags, libraries etc) + - name: Set up platformio environment + run: pio init --ide atom + + - name: Register problem matchers + run: | + echo "::add-matcher::.github/workflows/matchers/ci-custom.json" + echo "::add-matcher::.github/workflows/matchers/clang-tidy.json" + echo "::add-matcher::.github/workflows/matchers/gcc.json" + echo "::add-matcher::.github/workflows/matchers/lint-python.json" + echo "::add-matcher::.github/workflows/matchers/python.json" + - name: Run a quick lint over all changed files + run: script/quicklint + - name: Suggest changes + run: script/ci-suggest-changes + + lint-clang-format: + runs-on: ubuntu-latest + # cpp lint job runs with esphome-lint docker image so that clang-format-* + # doesn't have to be installed + container: esphome/esphome-lint:dev + steps: + - uses: actions/checkout@v2 + # Cache platformio intermediary files (like libraries etc) + # Note: platformio platform versions should be cached via the esphome-lint image + - name: Cache Platformio + uses: actions/cache@v1 + with: + path: .pio + key: lint-cpp-pio-${{ hashFiles('platformio.ini') }} + restore-keys: | + lint-cpp-pio- + # Set up the pio project so that the cpp checks know how files are compiled + # (build flags, libraries etc) + - name: Set up platformio environment + run: pio init --ide atom + + - name: Run clang-format + run: script/clang-format -i + - name: Suggest changes + run: script/ci-suggest-changes + + lint-clang-tidy: + runs-on: ubuntu-latest + # cpp lint job runs with esphome-lint docker image so that clang-format-* + # doesn't have to be installed + container: esphome/esphome-lint:dev + # Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files + strategy: + matrix: + split: [1, 2, 3, 4] + steps: + - uses: actions/checkout@v2 + # Cache platformio intermediary files (like libraries etc) + # Note: platformio platform versions should be cached via the esphome-lint image + - name: Cache Platformio + uses: actions/cache@v1 + with: + path: .pio + key: lint-cpp-pio-${{ hashFiles('platformio.ini') }} + restore-keys: | + lint-cpp-pio- + # Set up the pio project so that the cpp checks know how files are compiled + # (build flags, libraries etc) + - name: Set up platformio environment + run: pio init --ide atom + + + - name: Register problem matchers + run: | + echo "::add-matcher::.github/workflows/matchers/clang-tidy.json" + echo "::add-matcher::.github/workflows/matchers/gcc.json" + - name: Run clang-tidy + run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }} + - name: Suggest changes + run: script/ci-suggest-changes + + lint-python: + # Don't use the esphome-lint docker image because it may contain outdated requirements. + # This way, all dependencies are cached via the cache action. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Cache pip modules + uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: esphome-pip-3.7-${{ hashFiles('setup.py') }} + restore-keys: | + esphome-pip-3.7- + - name: Set up python environment + run: script/setup + + - name: Register problem matchers + run: | + echo "::add-matcher::.github/workflows/matchers/ci-custom.json" + echo "::add-matcher::.github/workflows/matchers/lint-python.json" + echo "::add-matcher::.github/workflows/matchers/python.json" + - name: Lint Custom + run: script/ci-custom.py + - name: Lint Python + run: script/lint-python + + test: + runs-on: ubuntu-latest + strategy: + matrix: + test: + - test1 + - test2 + - test3 + - test4 + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Cache pip modules + uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: esphome-pip-3.7-${{ hashFiles('setup.py') }} + restore-keys: | + esphome-pip-3.7- + # Use per test platformio cache because tests have different platform versions + - name: Cache ~/.platformio + uses: actions/cache@v1 + with: + path: ~/.platformio + key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }} + restore-keys: | + test-home-platformio-${{ matrix.test }}- + # Cache the intermediary build files + - name: Cache Test Build + uses: actions/cache@v1 + with: + path: tests/build/${{ matrix.test }} + key: test-pio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}-${{ hashFiles('esphome/**') }} + restore-keys: | + test-pio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}- + test-pio-${{ matrix.test }}- + - name: Set up environment + run: script/setup + + + - name: Register problem matchers + run: | + echo "::add-matcher::.github/workflows/matchers/gcc.json" + echo "::add-matcher::.github/workflows/matchers/python.json" + - run: esphome tests/${{ matrix.test }}.yaml compile diff --git a/.github/workflows/matchers/ci-custom.json b/.github/workflows/matchers/ci-custom.json new file mode 100644 index 0000000000..4e1eafff5e --- /dev/null +++ b/.github/workflows/matchers/ci-custom.json @@ -0,0 +1,16 @@ +{ + "problemMatcher": [ + { + "owner": "ci-custom", + "pattern": [ + { + "regexp": "^ERROR (.*):(\\d+):(\\d+) - (.*)$", + "file": 1, + "line": 2, + "column": 3, + "message": 4 + } + ] + } + ] +} diff --git a/.github/workflows/matchers/clang-tidy.json b/.github/workflows/matchers/clang-tidy.json new file mode 100644 index 0000000000..03e77341a5 --- /dev/null +++ b/.github/workflows/matchers/clang-tidy.json @@ -0,0 +1,17 @@ +{ + "problemMatcher": [ + { + "owner": "clang-tidy", + "pattern": [ + { + "regexp": "^(.*):(\\d+):(\\d+):\\s+(error):\\s+(.*) \\[([a-z0-9,\\-]+)\\]\\s*$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + } + ] + } + ] +} diff --git a/.github/workflows/matchers/gcc.json b/.github/workflows/matchers/gcc.json new file mode 100644 index 0000000000..899239f816 --- /dev/null +++ b/.github/workflows/matchers/gcc.json @@ -0,0 +1,18 @@ +{ + "problemMatcher": [ + { + "owner": "gcc", + "severity": "error", + "pattern": [ + { + "regexp": "^(.*):(\\d+):(\\d+):\\s+(?:fatal\\s+)?(warning|error):\\s+(.*)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + } + ] + } + ] +} diff --git a/.github/workflows/matchers/lint-python.json b/.github/workflows/matchers/lint-python.json new file mode 100644 index 0000000000..decbe36c4a --- /dev/null +++ b/.github/workflows/matchers/lint-python.json @@ -0,0 +1,28 @@ +{ + "problemMatcher": [ + { + "owner": "flake8", + "severity": "error", + "pattern": [ + { + "regexp": "^(.*):(\\d+) - ([EFCDNW]\\d{3}.*)$", + "file": 1, + "line": 2, + "message": 3 + } + ] + }, + { + "owner": "pylint", + "severity": "error", + "pattern": [ + { + "regexp": "^(.*):(\\d+) - (\\[[EFCRW]\\d{4}\\(.*\\),.*\\].*)$", + "file": 1, + "line": 2, + "message": 3 + } + ] + } + ] +} diff --git a/.github/workflows/matchers/python.json b/.github/workflows/matchers/python.json new file mode 100644 index 0000000000..9c3095c0c9 --- /dev/null +++ b/.github/workflows/matchers/python.json @@ -0,0 +1,18 @@ +{ + "problemMatcher": [ + { + "owner": "python", + "pattern": [ + { + "regexp": "^\\s*File\\s\\\"(.*)\\\",\\sline\\s(\\d+),\\sin\\s(.*)$", + "file": 1, + "line": 2 + }, + { + "regexp": "^\\s*raise\\s(.*)\\(\\'(.*)\\'\\)$", + "message": 2 + } + ] + } + ] +} diff --git a/.github/workflows/pull.yaml b/.github/workflows/pull.yaml deleted file mode 100644 index 6b66192150..0000000000 --- a/.github/workflows/pull.yaml +++ /dev/null @@ -1,59 +0,0 @@ -name: PR testing - -on: [pull_request] - -jobs: - lint-custom: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - name: Lint Custom - steps: - - uses: actions/checkout@v2 - - run: script/setup - - run: script/ci-custom.py - lint-python: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - name: Lint Python - steps: - - uses: actions/checkout@v2 - - run: script/setup - - run: script/lint-python - lint-tidy: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - name: Lint Tidy - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - run: script/setup - - run: pio init --ide atom - - run: script/clang-tidy --all-headers --fix - - run: script/ci-suggest-changes - lint-format: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - name: Lint Format - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - run: script/setup - - run: script/clang-format -i - - run: script/ci-suggest-changes - - test: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - strategy: - matrix: - test: - - test1 - - test2 - - test3 - - test4 - steps: - - uses: actions/checkout@v2 - - run: script/setup - - run: esphome tests/${{ matrix.test }}.yaml compile diff --git a/.github/workflows/release-dev.yaml b/.github/workflows/release-dev.yaml deleted file mode 100644 index 7f71d79610..0000000000 --- a/.github/workflows/release-dev.yaml +++ /dev/null @@ -1,124 +0,0 @@ -name: Release dev - -on: - push: - branches: - - dev - -jobs: - lint-custom: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - name: Lint Custom - steps: - - uses: actions/checkout@v2 - - run: script/setup - - run: script/ci-custom.py - lint-python: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - name: Lint Python - steps: - - uses: actions/checkout@v2 - - run: script/setup - - run: script/lint-python - lint-tidy: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - name: Lint Tidy - steps: - - uses: actions/checkout@v2 - - run: script/setup - - run: pio init --ide atom - - run: script/clang-tidy --all-headers --fix - - run: script/ci-suggest-changes - lint-format: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - name: Lint Format - steps: - - uses: actions/checkout@v2 - - run: script/setup - - run: script/clang-format -i - - run: script/ci-suggest-changes - - test: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - strategy: - matrix: - test: - - test1 - - test2 - - test3 - - test4 - steps: - - uses: actions/checkout@v2 - - run: script/setup - - run: esphome tests/${{ matrix.test }}.yaml compile - - deploy-docker: - runs-on: ubuntu-latest - needs: [lint-custom, lint-python, lint-tidy, lint-format, test] - strategy: - matrix: - arch: [aarch64, amd64, armv7, i386] - build-type: [hassio, docker] - steps: - - uses: actions/checkout@v2 - - run: docker info - - run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}" - env: - DOCKER_USER: ${{ secrets.DOCKER_USER }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - - run: docker run --rm --privileged multiarch/qemu-user-static:5.0.0-2 --reset -p yes - - env: - BASE_VERSION: 2.1.1 - run: | - if [[ "${{ matrix.build-type }}" == "hassio" ]]; then - BUILD_FROM=esphome/esphome-hassio-base-${{ matrix.arch }}:${BASE_VERSION} - BUILD_TO=${{ github.repository }}-hassio-${{ matrix.arch }} - DOCKERFILE=docker/Dockerfile.hassio - else - BUILD_FROM=esphome/esphome-base-${{ matrix.arch }}:${BASE_VERSION} - BUILD_TO=${{ github.repository }}-${{ matrix.arch }} - DOCKERFILE=docker/Dockerfile - fi - - TAG=${{ github.sha }} - TAG=${TAG:0:7} - - echo "Building tag: ${TAG}" - - docker build \ - --build-arg BUILD_FROM=${BUILD_FROM} \ - --build-arg BUILD_VERSION=${TAG} \ - --tag ${BUILD_TO}:dev \ - --file ${DOCKERFILE} \ - . - - echo "Pushing to ${BUILD_TO}:dev" - docker push ${BUILD_TO}:dev - - deploy-docker-manifest-version: - runs-on: ubuntu-latest - needs: [deploy-docker] - steps: - - run: mkdir -p ~/.docker - - run: | - echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json - - run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}" - env: - DOCKER_USER: ${{ secrets.DOCKER_USER }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - - run: | - REPO=${{ github.repository }} - - docker manifest create ${REPO}:dev \ - ${REPO}-aarch64:dev \ - ${REPO}-amd64:dev \ - ${REPO}-armv7:dev \ - ${REPO}-i386:dev - - echo "Pushing to ${REPO}:dev" - docker manifest push ${REPO}:dev diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml new file mode 100644 index 0000000000..564d12be88 --- /dev/null +++ b/.github/workflows/release-dev.yml @@ -0,0 +1,226 @@ +name: Publish dev releases to docker hub + +on: + push: + branches: + - dev + +jobs: + # THE LINT/TEST JOBS ARE COPIED FROM ci.yaml + + lint-clang-format: + runs-on: ubuntu-latest + # cpp lint job runs with esphome-lint docker image so that clang-format-* + # doesn't have to be installed + container: esphome/esphome-lint:dev + steps: + - uses: actions/checkout@v2 + # Cache platformio intermediary files (like libraries etc) + # Note: platformio platform versions should be cached via the esphome-lint image + - name: Cache Platformio + uses: actions/cache@v1 + with: + path: .pio + key: lint-cpp-pio-${{ hashFiles('platformio.ini') }} + restore-keys: | + lint-cpp-pio- + # Set up the pio project so that the cpp checks know how files are compiled + # (build flags, libraries etc) + - name: Set up platformio environment + run: pio init --ide atom + + - name: Run clang-format + run: script/clang-format -i + - name: Suggest changes + run: script/ci-suggest-changes + + lint-clang-tidy: + runs-on: ubuntu-latest + # cpp lint job runs with esphome-lint docker image so that clang-format-* + # doesn't have to be installed + container: esphome/esphome-lint:dev + # Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files + strategy: + matrix: + split: [1, 2, 3, 4] + steps: + - uses: actions/checkout@v2 + # Cache platformio intermediary files (like libraries etc) + # Note: platformio platform versions should be cached via the esphome-lint image + - name: Cache Platformio + uses: actions/cache@v1 + with: + path: .pio + key: lint-cpp-pio-${{ hashFiles('platformio.ini') }} + restore-keys: | + lint-cpp-pio- + # Set up the pio project so that the cpp checks know how files are compiled + # (build flags, libraries etc) + - name: Set up platformio environment + run: pio init --ide atom + + + - name: Register problem matchers + run: | + echo "::add-matcher::.github/workflows/matchers/clang-tidy.json" + echo "::add-matcher::.github/workflows/matchers/gcc.json" + - name: Run clang-tidy + run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }} + - name: Suggest changes + run: script/ci-suggest-changes + + lint-python: + # Don't use the esphome-lint docker image because it may contain outdated requirements. + # This way, all dependencies are cached via the cache action. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Cache pip modules + uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: esphome-pip-3.7-${{ hashFiles('setup.py') }} + restore-keys: | + esphome-pip-3.7- + - name: Set up python environment + run: script/setup + + - name: Register problem matchers + run: | + echo "::add-matcher::.github/workflows/matchers/ci-custom.json" + echo "::add-matcher::.github/workflows/matchers/lint-python.json" + echo "::add-matcher::.github/workflows/matchers/python.json" + - name: Lint Custom + run: script/ci-custom.py + - name: Lint Python + run: script/lint-python + + test: + runs-on: ubuntu-latest + strategy: + matrix: + test: + - test1 + - test2 + - test3 + - test4 + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Cache pip modules + uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: esphome-pip-3.7-${{ hashFiles('setup.py') }} + restore-keys: | + esphome-pip-3.7- + # Use per test platformio cache because tests have different platform versions + - name: Cache ~/.platformio + uses: actions/cache@v1 + with: + path: ~/.platformio + key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }} + restore-keys: | + test-home-platformio-${{ matrix.test }}- + # Cache the intermediary build files + - name: Cache Test Build + uses: actions/cache@v1 + with: + path: tests/build/${{ matrix.test }} + key: test-pio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}-${{ hashFiles('esphome/**') }} + restore-keys: | + test-pio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}- + test-pio-${{ matrix.test }}- + - name: Set up environment + run: script/setup + + + - name: Register problem matchers + run: | + echo "::add-matcher::.github/workflows/matchers/gcc.json" + echo "::add-matcher::.github/workflows/matchers/python.json" + - run: esphome tests/${{ matrix.test }}.yaml compile + + deploy-docker: + name: Build and publish docker containers + runs-on: ubuntu-latest + needs: [lint-clang-format, lint-clang-tidy, lint-python, test] + strategy: + matrix: + arch: [amd64, i386, armv7, aarch64] + build_type: ["hassio", "docker"] + steps: + - uses: actions/checkout@v2 + - name: Set up env variables + run: | + tag="dev" + base_version="2.1.2" + + if [[ "${{ matrix.build_type }}" == "hassio" ]]; then + build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}" + build_to="esphome/esphome-hassio-${{ matrix.arch }}" + dockerfile="docker/Dockerfile.hassio" + else + build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}" + build_to="esphome/esphome-${{ matrix.arch }}" + dockerfile="docker/Dockerfile" + if [[ "${{ matrix.arch }}" == "amd64" ]]; then + build_to="esphome/esphome" + fi + fi + + # Set env variables so these values don't need to be calculated again + echo "::set-env name=TAG::${tag}" + echo "::set-env name=BUILD_FROM::${build_from}" + echo "::set-env name=BUILD_TO::${build_to}" + echo "::set-env name=IMAGE::${build_to}:${tag}" + echo "::set-env name=DOCKERFILE::${dockerfile}" + - name: Register QEMU binfmt + run: docker run --rm --privileged multiarch/qemu-user-static:5.0.0-2 --reset -p yes + - run: | + docker build \ + --build-arg "BUILD_FROM=${BUILD_FROM}" \ + --build-arg "BUILD_VERSION=${TAG}" \ + --tag "${IMAGE}" \ + --file "${DOCKERFILE}" \ + . + - name: Log in to docker hub + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: docker login -u "$DOCKER_USER" -p "$DOCKER_PASSWORD" + - run: docker push "${IMAGE}" + + deploy-docker-manifest: + runs-on: ubuntu-latest + needs: [deploy-docker] + steps: + - name: Enable experimental manifest support + run: | + mkdir -p ~/.docker + echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json + - name: Log in to docker hub + env: + DOCKER_USER: ${{ secrets.DOCKER_USER }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}" + - name: "Create the manifest" + run: | + REPO=esphome/esphome + TAG="dev" + + docker manifest create ${REPO}:${TAG} \ + ${REPO}-aarch64:${TAG} \ + ${REPO}-amd64:${TAG} \ + ${REPO}-armv7:${TAG} \ + ${REPO}-i386:${TAG} + echo "::set-env name=TAG::${TAG}" + echo "::set-env name=REPO::${REPO}" + - run: docker push ${REPO}:${TAG} diff --git a/.github/workflows/release-version.yaml b/.github/workflows/release-version.yaml deleted file mode 100644 index cb72aae761..0000000000 --- a/.github/workflows/release-version.yaml +++ /dev/null @@ -1,180 +0,0 @@ -name: Release a version - -on: - push: - tags: - - "*" - -jobs: - lint-custom: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - name: Lint Custom - steps: - - uses: actions/checkout@v2 - - run: script/setup - - run: script/ci-custom.py - lint-python: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - name: Lint Python - steps: - - uses: actions/checkout@v2 - - run: script/setup - - run: script/lint-python - lint-tidy: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - name: Lint Tidy - steps: - - uses: actions/checkout@v2 - - run: script/setup - - run: pio init --ide atom - - run: script/clang-tidy --all-headers --fix - - run: script/ci-suggest-changes - lint-format: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - name: Lint Format - steps: - - uses: actions/checkout@v2 - - run: script/setup - - run: script/clang-format -i - - run: script/ci-suggest-changes - - test: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - strategy: - matrix: - test: - - test1 - - test2 - - test3 - - test4 - steps: - - uses: actions/checkout@v2 - - run: script/setup - - run: esphome tests/${{ matrix.test }}.yaml compile - - deploy-pypi: - runs-on: ubuntu-latest - container: jesserockz/esphome-lint - needs: [lint-custom, lint-python, lint-tidy, lint-format, test] - steps: - - run: pip install twine wheel - - run: python setup.py sdist bdist_wheel - - run: twine upload dist/* - env: - TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - - deploy-docker: - runs-on: ubuntu-latest - needs: [lint-custom, lint-python, lint-tidy, lint-format, test] - strategy: - matrix: - arch: [aarch64, amd64, armv7, i386] - build-type: [hassio, docker] - steps: - - uses: actions/checkout@v2 - - run: docker info - - run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}" - env: - DOCKER_USER: ${{ secrets.DOCKER_USER }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - - run: docker run --rm --privileged multiarch/qemu-user-static:5.0.0-2 --reset -p yes - - env: - BASE_VERSION: 2.1.1 - run: | - if [[ "${{ matrix.build-type }}" == "hassio" ]]; then - BUILD_FROM=esphome/esphome-hassio-base-${{ matrix.arch }}:${BASE_VERSION} - BUILD_TO=${{ github.repository }}-hassio-${{ matrix.arch }} - DOCKERFILE=docker/Dockerfile.hassio - else - BUILD_FROM=esphome/esphome-base-${{ matrix.arch }}:${BASE_VERSION} - BUILD_TO=${{ github.repository }}-${{ matrix.arch }} - DOCKERFILE=docker/Dockerfile - fi - - TAG=${{ github.ref }} - TAG=${TAG#refs/tags/v} - - echo "Building tag: ${TAG}" - - docker build \ - --build-arg BUILD_FROM=${BUILD_FROM} \ - --build-arg BUILD_VERSION=${TAG} \ - --tag ${BUILD_TO}:${TAG} \ - --file ${DOCKERFILE} \ - . - - echo "Pushing to ${BUILD_TO}:${TAG}" - docker push ${BUILD_TO}:${TAG} - - beta_tag="^v\d+\.\d+\.\d+b\d+$" - if [[ "${TAG}" ~= "${beta_tag}" ]]; then - echo "Pushing to ${BUILD_TO}:beta" - docker tag ${BUILD_TO}:${TAG} ${BUILD_TO}:beta - docker push ${BUILD_TO}:beta - else - echo "Pushing to ${BUILD_TO}:latest" - docker tag ${BUILD_TO}:${TAG} ${BUILD_TO}:latest - docker push ${BUILD_TO}:latest - fi - - deploy-docker-manifest-version: - runs-on: ubuntu-latest - steps: - - run: mkdir -p ~/.docker - - run: | - echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json - - run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}" - env: - DOCKER_USER: ${{ secrets.DOCKER_USER }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - - run: | - REPO=${{ github.repository }} - - TAG=${{ github.ref }} - TAG=${TAG#refs/tags/v} - - docker manifest create ${REPO}:${TAG} \ - ${REPO}-aarch64:${TAG} \ - ${REPO}-amd64:${TAG} \ - ${REPO}-armv7:${TAG} \ - ${REPO}-i386:${TAG} - - echo "Pushing to ${REPO}:${TAG}" - docker push ${REPO}:${TAG} - - deploy-docker-manifest: - runs-on: ubuntu-latest - steps: - - run: mkdir -p ~/.docker - - run: | - echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json - - run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}" - env: - DOCKER_USER: ${{ secrets.DOCKER_USER }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - - run: | - REPO=${{ github.repository }} - - TAG=${{ github.ref }} - TAG=${TAG#refs/tags/v} - - beta_tag="^v\d+\.\d+\.\d+b\d+$" - if [[ "${TAG}" ~= "${beta_tag}" ]]; then - TAG=beta - else - TAG=latest - fi - docker manifest create ${REPO}:${TAG} \ - ${REPO}-aarch64:${TAG} \ - ${REPO}-amd64:${TAG} \ - ${REPO}-armv7:${TAG} \ - ${REPO}-i386:${TAG} - - echo "Pushing to ${REPO}:${TAG}" - docker push ${REPO}:${TAG} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..6425cd8990 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,276 @@ +name: Publish Release + +on: + release: + types: [published] + +jobs: + # THE LINT/TEST JOBS ARE COPIED FROM ci.yaml + + lint-clang-format: + runs-on: ubuntu-latest + # cpp lint job runs with esphome-lint docker image so that clang-format-* + # doesn't have to be installed + container: esphome/esphome-lint:dev + steps: + - uses: actions/checkout@v2 + # Cache platformio intermediary files (like libraries etc) + # Note: platformio platform versions should be cached via the esphome-lint image + - name: Cache Platformio + uses: actions/cache@v1 + with: + path: .pio + key: lint-cpp-pio-${{ hashFiles('platformio.ini') }} + restore-keys: | + lint-cpp-pio- + # Set up the pio project so that the cpp checks know how files are compiled + # (build flags, libraries etc) + - name: Set up platformio environment + run: pio init --ide atom + + - name: Run clang-format + run: script/clang-format -i + - name: Suggest changes + run: script/ci-suggest-changes + + lint-clang-tidy: + runs-on: ubuntu-latest + # cpp lint job runs with esphome-lint docker image so that clang-format-* + # doesn't have to be installed + container: esphome/esphome-lint:dev + # Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files + strategy: + matrix: + split: [1, 2, 3, 4] + steps: + - uses: actions/checkout@v2 + # Cache platformio intermediary files (like libraries etc) + # Note: platformio platform versions should be cached via the esphome-lint image + - name: Cache Platformio + uses: actions/cache@v1 + with: + path: .pio + key: lint-cpp-pio-${{ hashFiles('platformio.ini') }} + restore-keys: | + lint-cpp-pio- + # Set up the pio project so that the cpp checks know how files are compiled + # (build flags, libraries etc) + - name: Set up platformio environment + run: pio init --ide atom + + + - name: Register problem matchers + run: | + echo "::add-matcher::.github/workflows/matchers/clang-tidy.json" + echo "::add-matcher::.github/workflows/matchers/gcc.json" + - name: Run clang-tidy + run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }} + - name: Suggest changes + run: script/ci-suggest-changes + + lint-python: + # Don't use the esphome-lint docker image because it may contain outdated requirements. + # This way, all dependencies are cached via the cache action. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Cache pip modules + uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: esphome-pip-3.7-${{ hashFiles('setup.py') }} + restore-keys: | + esphome-pip-3.7- + - name: Set up python environment + run: script/setup + + - name: Register problem matchers + run: | + echo "::add-matcher::.github/workflows/matchers/ci-custom.json" + echo "::add-matcher::.github/workflows/matchers/lint-python.json" + echo "::add-matcher::.github/workflows/matchers/python.json" + - name: Lint Custom + run: script/ci-custom.py + - name: Lint Python + run: script/lint-python + + test: + runs-on: ubuntu-latest + strategy: + matrix: + test: + - test1 + - test2 + - test3 + - test4 + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Cache pip modules + uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: esphome-pip-3.7-${{ hashFiles('setup.py') }} + restore-keys: | + esphome-pip-3.7- + # Use per test platformio cache because tests have different platform versions + - name: Cache ~/.platformio + uses: actions/cache@v1 + with: + path: ~/.platformio + key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }} + restore-keys: | + test-home-platformio-${{ matrix.test }}- + # Cache the intermediary build files + - name: Cache Test Build + uses: actions/cache@v1 + with: + path: tests/build/${{ matrix.test }} + key: test-pio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}-${{ hashFiles('esphome/**') }} + restore-keys: | + test-pio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}- + test-pio-${{ matrix.test }}- + - name: Set up environment + run: script/setup + + + - name: Register problem matchers + run: | + echo "::add-matcher::.github/workflows/matchers/gcc.json" + echo "::add-matcher::.github/workflows/matchers/python.json" + - run: esphome tests/${{ matrix.test }}.yaml compile + + deploy-pypi: + name: Build and publish to PyPi + needs: [lint-clang-format, lint-clang-tidy, lint-python, test] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: '3.x' + - name: Set up python environment + run: | + script/setup + pip install setuptools wheel twine + - name: Build + run: python setup.py sdist bdist_wheel + - name: Upload + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: twine upload dist/* + + deploy-docker: + name: Build and publish docker containers + runs-on: ubuntu-latest + needs: [lint-clang-format, lint-clang-tidy, lint-python, test] + strategy: + matrix: + arch: [amd64, i386, armv7, aarch64] + build_type: ["hassio", "docker"] + steps: + - uses: actions/checkout@v2 + - name: Set up env variables + run: | + tag="${GITHUB_REF#v}" + base_version="2.1.2" + + if [[ "${{ matrix.build_type }}" == "hassio" ]]; then + build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}" + build_to="esphome/esphome-hassio-${{ matrix.arch }}" + dockerfile="docker/Dockerfile.hassio" + else + build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}" + build_to="esphome/esphome-${{ matrix.arch }}" + dockerfile="docker/Dockerfile" + if [[ "${{ matrix.arch }}" == "amd64" ]]; then + build_to="esphome/esphome" + fi + fi + + # Set env variables so these values don't need to be calculated again + echo "::set-env name=TAG::${tag}" + echo "::set-env name=BUILD_FROM::${build_from}" + echo "::set-env name=BUILD_TO::${build_to}" + echo "::set-env name=IMAGE::${build_to}:${tag}" + echo "::set-env name=DOCKERFILE::${dockerfile}" + - name: Register QEMU binfmt + run: docker run --rm --privileged multiarch/qemu-user-static:5.0.0-2 --reset -p yes + - run: | + docker build \ + --build-arg "BUILD_FROM=${BUILD_FROM}" \ + --build-arg "BUILD_VERSION=${TAG}" \ + --tag "${IMAGE}" \ + --file "${DOCKERFILE}" \ + . + - name: Log in to docker hub + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: docker login -u "$DOCKER_USER" -p "$DOCKER_PASSWORD" + - run: docker push "${IMAGE}" + - if: ${{ github.event.release.prerelease) }} + name: Publish docker beta tag + run: | + docker tag "${IMAGE}" ${BUILD_TO}:beta + docker push "${BUILD_TO}:beta" + - if: ${{ !github.event.release.prerelease) }} + name: Publish docker latest tag + run: | + docker tag "${IMAGE}" ${BUILD_TO}:latest + docker push "${BUILD_TO}:latest" + + deploy-docker-manifest: + runs-on: ubuntu-latest + needs: [deploy-docker] + steps: + - name: Enable experimental manifest support + run: | + mkdir -p ~/.docker + echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json + - name: Log in to docker hub + env: + DOCKER_USER: ${{ secrets.DOCKER_USER }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}" + - name: "Create the manifest" + run: | + REPO=esphome/esphome + TAG="${GITHUB_REF#v}" + + docker manifest create ${REPO}:${TAG} \ + ${REPO}-aarch64:${TAG} \ + ${REPO}-amd64:${TAG} \ + ${REPO}-armv7:${TAG} \ + ${REPO}-i386:${TAG} + echo "::set-env name=TAG::${TAG}" + echo "::set-env name=REPO::${REPO}" + - run: docker push ${REPO}:${TAG} + + - name: Publish docker beta tag + if: ${{ github.event.release.prerelease) }} + run: | + docker manifest create ${REPO}:beta \ + ${REPO}-aarch64:beta \ + ${REPO}-amd64:beta \ + ${REPO}-armv7:beta \ + ${REPO}-i386:beta + docker push ${REPO}:beta + - name: Publish docker latest tag + if: ${{ !github.event.release.prerelease) }} + run: | + docker manifest create ${REPO}:latest \ + ${REPO}-aarch64:latest \ + ${REPO}-amd64:latest \ + ${REPO}-armv7:latest \ + ${REPO}-i386:latest + docker push ${REPO}:latest diff --git a/docker/Dockerfile.lint b/docker/Dockerfile.lint index cdd715ef5a..32b316075f 100644 --- a/docker/Dockerfile.lint +++ b/docker/Dockerfile.lint @@ -1,4 +1,4 @@ -FROM esphome/esphome-base-amd64:2.1.1 +FROM esphome/esphome-base-amd64:2.1.2 RUN \ apt-get update \ @@ -7,6 +7,8 @@ RUN \ clang-tidy-7 \ patch \ software-properties-common \ + # Update to latest git version because of github actions + # https://github.com/actions/checkout/issues/126 && apt-add-repository ppa:git-core/ppa \ && apt-get install -y --no-install-recommends \ git \ @@ -16,7 +18,7 @@ RUN \ /var/lib/apt/lists/* COPY requirements_test.txt /requirements_test.txt -RUN pip3 install --no-cache-dir wheel && pip3 install --no-cache-dir -r /requirements_test.txt +RUN pip3 install --no-cache-dir -r /requirements_test.txt VOLUME ["/esphome"] WORKDIR /esphome diff --git a/script/ci-custom.py b/script/ci-custom.py index b2b838cb5b..67010d77f9 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -101,10 +101,12 @@ def lint_re_check(regex, **kwargs): if 'NOLINT' in match.group(0): continue lineno = content.count("\n", 0, match.start()) + 1 + substr = content[:match.start()] + col = len(substr) - substr.rfind('\n') err = func(fname, match) if err is None: continue - errors.append(f"{err} See line {lineno}.") + errors.append((lineno, col+1, err)) return errors return decor(new_func) return decorator @@ -121,8 +123,7 @@ def lint_content_find_check(find, **kwargs): errors = [] for line, col in find_all(content, find_): err = func(fname) - errors.append("{err} See line {line}:{col}." - "".format(err=err, line=line+1, col=col+1)) + errors.append((line+1, col+1, err)) return errors return decor(new_func) return decorator @@ -215,9 +216,10 @@ def lint_const_ordered(fname, content): continue target = next(i for i, l in ordered if l == ml) target_text = next(l for i, l in matching if target == i) - errors.append("Constant {} is not ordered, please make sure all constants are ordered. " - "See line {} (should go to line {}, {})" - "".format(highlight(ml), mi, target, target_text)) + errors.append((ml, None, + "Constant {} is not ordered, please make sure all constants are ordered. " + "See line {} (should go to line {}, {})" + "".format(highlight(ml), mi, target, target_text))) return errors @@ -354,13 +356,22 @@ errors = collections.defaultdict(list) def add_errors(fname, errs): if not isinstance(errs, list): errs = [errs] - errs = [x for x in errs if x is not None] for err in errs: + if err is None: + continue + try: + lineno, col, msg = err + except ValueError: + lineno = 1 + col = 1 + msg = err if not isinstance(err, str): raise ValueError("Error is not instance of string!") - if not errs: - return - errors[fname].extend(errs) + if not isinstance(lineno, int): + raise ValueError("Line number is not an int!") + if not isinstance(col, int): + raise ValueError("Column number is not an int!") + errors[fname].append((lineno, col, msg)) for fname in files: @@ -380,8 +391,8 @@ run_checks(LINT_POST_CHECKS, 'POST') for f, errs in sorted(errors.items()): print(f"\033[0;32m************* File \033[1;32m{f}\033[0m") - for err in errs: - print(err) + for lineno, col, msg in errs: + print(f"ERROR {f}:{lineno}:{col} - {msg}") print() sys.exit(len(errors)) diff --git a/script/clang-tidy b/script/clang-tidy index 1005d15580..490e63e0d2 100755 --- a/script/clang-tidy +++ b/script/clang-tidy @@ -2,20 +2,19 @@ from __future__ import print_function +import argparse import multiprocessing import os import re - -import pexpect import shutil import subprocess import sys import tempfile - -import argparse -import click import threading +import click +import pexpect + sys.path.append(os.path.dirname(__file__)) from helpers import basepath, shlex_quote, get_output, build_compile_commands, \ build_all_include, temp_header_file, git_ls_files, filter_changed @@ -49,7 +48,7 @@ def run_tidy(args, tmpdir, queue, lock, failed_files): # Use pexpect for a pseudy-TTY with colored output output, rc = pexpect.run(invocation_s, withexitstatus=True, encoding='utf-8', - timeout=15*60) + timeout=15 * 60) with lock: if rc != 0: print() @@ -65,6 +64,11 @@ def progress_bar_show(value): return '' +def split_list(a, n): + k, m = divmod(len(a), n) + return [a[i * k + min(i, m):(i + 1) * k + min(i + 1, m)] for i in range(n)] + + def main(): parser = argparse.ArgumentParser() parser.add_argument('-j', '--jobs', type=int, @@ -77,6 +81,10 @@ def main(): help='Run clang-tidy in quiet mode') parser.add_argument('-c', '--changed', action='store_true', help='Only run on changed files') + parser.add_argument('--split-num', type=int, help='Split the files into X jobs.', + default=None) + parser.add_argument('--split-at', type=int, help='Which split is this? Starts at 1', + default=None) parser.add_argument('--all-headers', action='store_true', help='Create a dummy file that checks all headers') args = parser.parse_args() @@ -114,7 +122,10 @@ def main(): files.sort() - if args.all_headers: + if args.split_num: + files = split_list(files, args.split_num)[args.split_at - 1] + + if args.all_headers and args.split_at in (None, 1): files.insert(0, temp_header_file) tmpdir = None @@ -157,8 +168,8 @@ def main(): print('Error applying fixes.\n', file=sys.stderr) raise - sys.exit(return_code) + return return_code if __name__ == '__main__': - main() + sys.exit(main()) diff --git a/script/helpers.py b/script/helpers.py index c9bf5224b1..e0aaee8711 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -101,8 +101,10 @@ def splitlines_no_ends(string): def changed_files(): - for remote in ('upstream', 'origin'): - command = ['git', 'merge-base', f'{remote}/dev', 'HEAD'] + check_remotes = ['upstream', 'origin'] + check_remotes.extend(splitlines_no_ends(get_output('git', 'remote'))) + for remote in check_remotes: + command = ['git', 'merge-base', f'refs/remotes/{remote}/dev', 'HEAD'] try: merge_base = splitlines_no_ends(get_output(*command))[0] break