From 4f9a56c8841a247f7d82fcd0cbe40902a77ba9e2 Mon Sep 17 00:00:00 2001 From: Otto Winter Date: Thu, 15 Jul 2021 21:30:04 +0200 Subject: [PATCH] Refactor docker build system and workflows (#2023) --- .github/workflows/ci-docker.yml | 47 ++-- .github/workflows/ci.yml | 178 ++++++------- .github/workflows/docker-lint-build.yml | 108 ++++++-- .github/workflows/matchers/pytest.json | 19 ++ .github/workflows/release-dev.yml | 247 ------------------- .github/workflows/release.yml | 315 ++++++------------------ docker/Dockerfile | 2 +- docker/Dockerfile.hassio | 2 +- docker/Dockerfile.lint | 3 +- docker/build.py | 177 +++++++++++++ esphome/const.py | 6 +- script/bump-docker-base-version.py | 50 ---- script/bump-version.py | 10 +- 13 files changed, 456 insertions(+), 708 deletions(-) create mode 100644 .github/workflows/matchers/pytest.json delete mode 100644 .github/workflows/release-dev.yml create mode 100755 docker/build.py delete mode 100755 script/bump-docker-base-version.py diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index 91ec88aeb3..45fd3e141b 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -18,38 +18,23 @@ jobs: name: Build docker containers runs-on: ubuntu-latest strategy: - fail-fast: false matrix: arch: [amd64, armv7, aarch64] - build_type: ["hassio", "docker"] + build_type: ["ha-addon", "docker", "lint"] steps: - - uses: actions/checkout@v2 - - name: Set up env variables - run: | - base_version="3.4.0" + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + - name: Set TAG + run: | + echo "TAG=check" >> $GITHUB_ENV - if [[ "${{ matrix.build_type }}" == "hassio" ]]; then - build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}" - build_to="esphome/esphome-hassio-${{ matrix.arch }}" - dockerfile="docker/Dockerfile.hassio" - else - build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}" - build_to="esphome/esphome-${{ matrix.arch }}" - dockerfile="docker/Dockerfile" - fi - - echo "BUILD_FROM=${build_from}" >> $GITHUB_ENV - echo "BUILD_TO=${build_to}" >> $GITHUB_ENV - echo "DOCKERFILE=${dockerfile}" >> $GITHUB_ENV - - name: Pull for cache - run: | - docker pull "${BUILD_TO}:dev" || true - - name: Register QEMU binfmt - run: docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset -p yes - - run: | - docker build \ - --build-arg "BUILD_FROM=${BUILD_FROM}" \ - --build-arg "BUILD_VERSION=ci" \ - --cache-from "${BUILD_TO}:dev" \ - --file "${DOCKERFILE}" \ - . + - name: Run build + run: | + docker/build.py \ + --tag "${TAG}" \ + --arch "${{ matrix.arch }}" \ + --build-type "${{ matrix.build_type }}" \ + build diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 121b3f1339..de777d5a3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,40 +4,36 @@ name: CI on: push: - # On dev branch release-dev already performs CI checks - # On other branches the `pull_request` trigger will be used - branches: [beta, release] + branches: [dev, beta, release] pull_request: jobs: - lint-clang-format: + ci-with-container: + name: ${{ matrix.name }} runs-on: ubuntu-latest - # cpp lint job runs with esphome-lint docker image so that clang-format-* - # doesn't have to be installed - container: esphome/esphome-lint:1.1 - steps: - - uses: actions/checkout@v2 - # Set up the pio project so that the cpp checks know how files are compiled - # (build flags, libraries etc) - - name: Set up platformio environment - run: pio init --ide atom - - - name: Run clang-format - run: script/clang-format -i - - name: Suggest changes - run: script/ci-suggest-changes - - lint-clang-tidy: - runs-on: ubuntu-latest - # cpp lint job runs with esphome-lint docker image so that clang-format-* - # doesn't have to be installed - container: esphome/esphome-lint:1.1 - # Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files strategy: fail-fast: false matrix: - split: [1, 2, 3, 4] + include: + - id: clang-format + name: Run script/clang-format + - id: clang-tidy + name: Run script/clang-tidy 1/4 + split: 1 + - id: clang-tidy + name: Run script/clang-tidy 2/4 + split: 2 + - id: clang-tidy + name: Run script/clang-tidy 3/4 + split: 3 + - id: clang-tidy + name: Run script/clang-tidy 4/4 + split: 4 + + # cpp lint job runs with esphome-lint docker image so that clang-format-* + # doesn't have to be installed + container: esphome/esphome-lint:1.1 steps: - uses: actions/checkout@v2 # Set up the pio project so that the cpp checks know how files are compiled @@ -45,26 +41,57 @@ jobs: - name: Set up platformio environment run: pio init --ide atom - - name: Register problem matchers run: | echo "::add-matcher::.github/workflows/matchers/clang-tidy.json" echo "::add-matcher::.github/workflows/matchers/gcc.json" + + - name: Run clang-format + run: script/clang-format -i + if: ${{ matrix.id == 'clang-format' }} + - name: Run clang-tidy run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }} + if: ${{ matrix.id == 'clang-tidy' }} + - name: Suggest changes run: script/ci-suggest-changes - lint-python: + ci: # Don't use the esphome-lint docker image because it may contain outdated requirements. # This way, all dependencies are cached via the cache action. + name: ${{ matrix.name }} runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - id: ci-custom + name: Run script/ci-custom + - id: lint-python + name: Run script/lint-python + - id: test + file: tests/test1.yaml + name: Test tests/test1.yaml + - id: test + file: tests/test2.yaml + name: Test tests/test2.yaml + - id: test + file: tests/test3.yaml + name: Test tests/test3.yaml + - id: test + file: tests/test4.yaml + name: Test tests/test4.yaml + - id: pytest + name: Run pytest + steps: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 with: python-version: '3.7' + - name: Cache pip modules uses: actions/cache@v1 with: @@ -72,6 +99,17 @@ jobs: key: esphome-pip-3.7-${{ hashFiles('setup.py') }} restore-keys: | esphome-pip-3.7- + + # Use per test platformio cache because tests have different platform versions + - name: Cache ~/.platformio + uses: actions/cache@v1 + with: + path: ~/.platformio + key: test-home-platformio-${{ matrix.file }}-${{ hashFiles('esphome/core/config.py') }} + restore-keys: | + test-home-platformio-${{ matrix.file }}- + if: ${{ matrix.id == 'test' }} + - name: Set up python environment run: script/setup @@ -80,82 +118,22 @@ jobs: echo "::add-matcher::.github/workflows/matchers/ci-custom.json" echo "::add-matcher::.github/workflows/matchers/lint-python.json" echo "::add-matcher::.github/workflows/matchers/python.json" + echo "::add-matcher::.github/workflows/matchers/pytest.json" + echo "::add-matcher::.github/workflows/matchers/gcc.json" + - name: Lint Custom - run: script/ci-custom.py + run: | + script/ci-custom.py + script/build_codeowners.py --check + if: ${{ matrix.id == 'ci-custom' }} - name: Lint Python run: script/lint-python - - name: Lint CODEOWNERS - run: script/build_codeowners.py --check + if: ${{ matrix.id == 'lint-python' }} - test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - test: - - test1 - - test2 - - test3 - - test4 - - test5 - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Cache pip modules - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: esphome-pip-3.7-${{ hashFiles('setup.py') }} - restore-keys: | - esphome-pip-3.7- - # Use per test platformio cache because tests have different platform versions - - name: Cache ~/.platformio - uses: actions/cache@v1 - with: - path: ~/.platformio - key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }} - restore-keys: | - test-home-platformio-${{ matrix.test }}- - - name: Set up environment - run: script/setup + - run: esphome compile ${{ matrix.file }} + if: ${{ matrix.id == 'test' }} - - - name: Register problem matchers - run: | - echo "::add-matcher::.github/workflows/matchers/gcc.json" - echo "::add-matcher::.github/workflows/matchers/python.json" - - run: esphome compile tests/${{ matrix.test }}.yaml - - pytest: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Cache pip modules - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: esphome-pip-3.7-${{ hashFiles('setup.py') }} - restore-keys: | - esphome-pip-3.7- - - name: Set up environment - run: script/setup - - name: Install Github Actions annotator - run: pip install pytest-github-actions-annotate-failures - - - name: Register problem matchers - run: | - echo "::add-matcher::.github/workflows/matchers/python.json" - name: Run pytest run: | - pytest \ - -qq \ - --durations=10 \ - -o console_output_style=count \ - tests + pytest -vv --tb=native tests + if: ${{ matrix.id == 'pytest' }} diff --git a/.github/workflows/docker-lint-build.yml b/.github/workflows/docker-lint-build.yml index d254ac332a..32aec87cdd 100644 --- a/.github/workflows/docker-lint-build.yml +++ b/.github/workflows/docker-lint-build.yml @@ -13,30 +13,88 @@ on: - '.github/workflows/docker-lint-build.yml' jobs: - publish-docker-lint-iage: - name: Build docker containers + deploy-docker: + name: Build and publish docker containers + if: github.repository == 'esphome/esphome' runs-on: ubuntu-latest + strategy: + matrix: + arch: [amd64, armv7, aarch64] + build_type: ["lint"] steps: - - uses: actions/checkout@v2 - - name: Set TAG - run: | - echo "TAG=1.1" >> $GITHUB_ENV - - name: Pull for cache - run: | - docker pull "esphome/esphome-lint:latest" || true - - name: Build - run: | - docker build \ - --cache-from "esphome/esphome-lint:latest" \ - --file "docker/Dockerfile.lint" \ - --tag "esphome/esphome-lint:latest" \ - --tag "esphome/esphome-lint:${TAG}" \ - . - - name: Log in to docker hub - env: - DOCKER_USER: ${{ secrets.DOCKER_USER }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}" - - run: | - docker push "esphome/esphome-lint:${TAG}" - docker push "esphome/esphome-lint:latest" + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + - name: Set TAG + run: | + echo "TAG=1.1" >> $GITHUB_ENV + + - name: Run build + run: | + docker/build.py \ + --tag "${TAG}" \ + --arch "${{ matrix.arch }}" \ + --build-type "${{ matrix.build_type }}" \ + build + + - name: Log in to docker hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Log in to the GitHub container registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Run push + run: | + docker/build.py \ + --tag "${TAG}" \ + --arch "${{ matrix.arch }}" \ + --build-type "${{ matrix.build_type }}" \ + push + + deploy-docker-manifest: + if: github.repository == 'esphome/esphome' + runs-on: ubuntu-latest + needs: [deploy-docker] + strategy: + matrix: + build_type: ["lint"] + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + - name: Set TAG + run: | + echo "TAG=1.1" >> $GITHUB_ENV + - name: Enable experimental manifest support + run: | + mkdir -p ~/.docker + echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json + + - name: Log in to docker hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Log in to the GitHub container registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Run manifest + run: | + docker/build.py \ + --tag "${TAG}" \ + --build-type "${{ matrix.build_type }}" \ + manifest diff --git a/.github/workflows/matchers/pytest.json b/.github/workflows/matchers/pytest.json new file mode 100644 index 0000000000..0eb8f050e6 --- /dev/null +++ b/.github/workflows/matchers/pytest.json @@ -0,0 +1,19 @@ +{ + "problemMatcher": [ + { + "owner": "pytest", + "fileLocation": "absolute", + "pattern": [ + { + "regexp": "^\\s+File \"(.*)\", line (\\d+), in (.*)$", + "file": 1, + "line": 2 + }, + { + "regexp": "^\\s+(.*)$", + "message": 1 + } + ] + } + ] +} diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml deleted file mode 100644 index f8b90d524f..0000000000 --- a/.github/workflows/release-dev.yml +++ /dev/null @@ -1,247 +0,0 @@ -name: Publish dev releases to docker hub - -on: - push: - branches: - - dev - -jobs: - # THE LINT/TEST JOBS ARE COPIED FROM ci.yaml - - lint-clang-format: - runs-on: ubuntu-latest - # cpp lint job runs with esphome-lint docker image so that clang-format-* - # doesn't have to be installed - container: esphome/esphome-lint:1.1 - steps: - - uses: actions/checkout@v2 - # Set up the pio project so that the cpp checks know how files are compiled - # (build flags, libraries etc) - - name: Set up platformio environment - run: pio init --ide atom - - - name: Run clang-format - run: script/clang-format -i - - name: Suggest changes - run: script/ci-suggest-changes - - lint-clang-tidy: - runs-on: ubuntu-latest - # cpp lint job runs with esphome-lint docker image so that clang-format-* - # doesn't have to be installed - container: esphome/esphome-lint:1.1 - # Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files - strategy: - fail-fast: false - matrix: - split: [1, 2, 3, 4] - steps: - - uses: actions/checkout@v2 - # Set up the pio project so that the cpp checks know how files are compiled - # (build flags, libraries etc) - - name: Set up platformio environment - run: pio init --ide atom - - - - name: Register problem matchers - run: | - echo "::add-matcher::.github/workflows/matchers/clang-tidy.json" - echo "::add-matcher::.github/workflows/matchers/gcc.json" - - name: Run clang-tidy - run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }} - - name: Suggest changes - run: script/ci-suggest-changes - - lint-python: - # Don't use the esphome-lint docker image because it may contain outdated requirements. - # This way, all dependencies are cached via the cache action. - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Cache pip modules - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: esphome-pip-3.7-${{ hashFiles('setup.py') }} - restore-keys: | - esphome-pip-3.7- - - name: Set up python environment - run: script/setup - - - name: Register problem matchers - run: | - echo "::add-matcher::.github/workflows/matchers/ci-custom.json" - echo "::add-matcher::.github/workflows/matchers/lint-python.json" - echo "::add-matcher::.github/workflows/matchers/python.json" - - name: Lint Custom - run: script/ci-custom.py - - name: Lint Python - run: script/lint-python - - name: Lint CODEOWNERS - run: script/build_codeowners.py --check - - test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - test: - - test1 - - test2 - - test3 - - test4 - - test5 - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Cache pip modules - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: esphome-pip-3.7-${{ hashFiles('setup.py') }} - restore-keys: | - esphome-pip-3.7- - # Use per test platformio cache because tests have different platform versions - - name: Cache ~/.platformio - uses: actions/cache@v1 - with: - path: ~/.platformio - key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }} - restore-keys: | - test-home-platformio-${{ matrix.test }}- - - name: Set up environment - run: script/setup - - - - name: Register problem matchers - run: | - echo "::add-matcher::.github/workflows/matchers/gcc.json" - echo "::add-matcher::.github/workflows/matchers/python.json" - - run: esphome compile tests/${{ matrix.test }}.yaml - - pytest: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Cache pip modules - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: esphome-pip-3.7-${{ hashFiles('setup.py') }} - restore-keys: | - esphome-pip-3.7- - - name: Set up environment - run: script/setup - - name: Install Github Actions annotator - run: pip install pytest-github-actions-annotate-failures - - - name: Register problem matchers - run: | - echo "::add-matcher::.github/workflows/matchers/python.json" - - name: Run pytest - run: | - pytest \ - -qq \ - --durations=10 \ - -o console_output_style=count \ - tests - - deploy-docker: - name: Build and publish docker containers - if: github.repository == 'esphome/esphome' - runs-on: ubuntu-latest - needs: [lint-clang-format, lint-clang-tidy, lint-python, test, pytest] - strategy: - matrix: - arch: [amd64, armv7, aarch64] - # Hassio dev image doesn't use esphome/esphome-hassio-$arch and uses base directly - build_type: ["docker"] - steps: - - uses: actions/checkout@v2 - - name: Set TAG - run: | - TAG="${GITHUB_SHA:0:7}" - echo "TAG=${TAG}" >> $GITHUB_ENV - - name: Set up env variables - run: | - base_version="3.4.0" - - if [[ "${{ matrix.build_type }}" == "hassio" ]]; then - build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}" - build_to="esphome/esphome-hassio-${{ matrix.arch }}" - dockerfile="docker/Dockerfile.hassio" - else - build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}" - build_to="esphome/esphome-${{ matrix.arch }}" - dockerfile="docker/Dockerfile" - fi - - echo "BUILD_FROM=${build_from}" >> $GITHUB_ENV - echo "BUILD_TO=${build_to}" >> $GITHUB_ENV - echo "DOCKERFILE=${dockerfile}" >> $GITHUB_ENV - - name: Pull for cache - run: | - docker pull "${BUILD_TO}:dev" || true - - name: Register QEMU binfmt - run: docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset -p yes - - run: | - docker build \ - --build-arg "BUILD_FROM=${BUILD_FROM}" \ - --build-arg "BUILD_VERSION=${TAG}" \ - --tag "${BUILD_TO}:${TAG}" \ - --tag "${BUILD_TO}:dev" \ - --cache-from "${BUILD_TO}:dev" \ - --file "${DOCKERFILE}" \ - . - - name: Log in to docker hub - env: - DOCKER_USER: ${{ secrets.DOCKER_USER }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}" - - run: | - docker push "${BUILD_TO}:${TAG}" - docker push "${BUILD_TO}:dev" - - - deploy-docker-manifest: - if: github.repository == 'esphome/esphome' - runs-on: ubuntu-latest - needs: [deploy-docker] - steps: - - name: Enable experimental manifest support - run: | - mkdir -p ~/.docker - echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json - - name: Set TAG - run: | - TAG="${GITHUB_SHA:0:7}" - echo "TAG=${TAG}" >> $GITHUB_ENV - - name: Log in to docker hub - env: - DOCKER_USER: ${{ secrets.DOCKER_USER }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}" - - name: "Create the manifest" - run: | - docker manifest create esphome/esphome:${TAG} \ - esphome/esphome-aarch64:${TAG} \ - esphome/esphome-amd64:${TAG} \ - esphome/esphome-armv7:${TAG} - docker manifest push esphome/esphome:${TAG} - - docker manifest create esphome/esphome:dev \ - esphome/esphome-aarch64:${TAG} \ - esphome/esphome-amd64:${TAG} \ - esphome/esphome-armv7:${TAG} - docker manifest push esphome/esphome:dev diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 383f2878e5..1b15b540f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,164 +1,35 @@ name: Publish Release on: + workflow_dispatch: release: types: [published] + schedule: + - cron: "0 2 * * *" jobs: - # THE LINT/TEST JOBS ARE COPIED FROM ci.yaml - - lint-clang-format: + init: + name: Initialize build runs-on: ubuntu-latest - # cpp lint job runs with esphome-lint docker image so that clang-format-* - # doesn't have to be installed - container: esphome/esphome-lint:1.1 + outputs: + tag: ${{ steps.tag.outputs.tag }} steps: - uses: actions/checkout@v2 - # Set up the pio project so that the cpp checks know how files are compiled - # (build flags, libraries etc) - - name: Set up platformio environment - run: pio init --ide atom - - - name: Run clang-format - run: script/clang-format -i - - name: Suggest changes - run: script/ci-suggest-changes - - lint-clang-tidy: - runs-on: ubuntu-latest - # cpp lint job runs with esphome-lint docker image so that clang-format-* - # doesn't have to be installed - container: esphome/esphome-lint:1.1 - # Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files - strategy: - fail-fast: false - matrix: - split: [1, 2, 3, 4] - steps: - - uses: actions/checkout@v2 - # Set up the pio project so that the cpp checks know how files are compiled - # (build flags, libraries etc) - - name: Set up platformio environment - run: pio init --ide atom - - - - name: Register problem matchers + - name: Get tag + id: tag run: | - echo "::add-matcher::.github/workflows/matchers/clang-tidy.json" - echo "::add-matcher::.github/workflows/matchers/gcc.json" - - name: Run clang-tidy - run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }} - - name: Suggest changes - run: script/ci-suggest-changes - - lint-python: - # Don't use the esphome-lint docker image because it may contain outdated requirements. - # This way, all dependencies are cached via the cache action. - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Cache pip modules - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: esphome-pip-3.7-${{ hashFiles('setup.py') }} - restore-keys: | - esphome-pip-3.7- - - name: Set up python environment - run: script/setup - - - name: Register problem matchers - run: | - echo "::add-matcher::.github/workflows/matchers/ci-custom.json" - echo "::add-matcher::.github/workflows/matchers/lint-python.json" - echo "::add-matcher::.github/workflows/matchers/python.json" - - name: Lint Custom - run: script/ci-custom.py - - name: Lint Python - run: script/lint-python - - name: Lint CODEOWNERS - run: script/build_codeowners.py --check - - test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - test: - - test1 - - test2 - - test3 - - test4 - - test5 - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Cache pip modules - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: esphome-pip-3.7-${{ hashFiles('setup.py') }} - restore-keys: | - esphome-pip-3.7- - # Use per test platformio cache because tests have different platform versions - - name: Cache ~/.platformio - uses: actions/cache@v1 - with: - path: ~/.platformio - key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }} - restore-keys: | - test-home-platformio-${{ matrix.test }}- - - name: Set up environment - run: script/setup - - - name: Register problem matchers - run: | - echo "::add-matcher::.github/workflows/matchers/gcc.json" - echo "::add-matcher::.github/workflows/matchers/python.json" - - run: esphome compile tests/${{ matrix.test }}.yaml - - pytest: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Cache pip modules - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: esphome-pip-3.7-${{ hashFiles('setup.py') }} - restore-keys: | - esphome-pip-3.7- - - name: Set up environment - run: script/setup - - name: Install Github Actions annotator - run: pip install pytest-github-actions-annotate-failures - - - name: Register problem matchers - run: | - echo "::add-matcher::.github/workflows/matchers/python.json" - - name: Run pytest - run: | - pytest \ - -qq \ - --durations=10 \ - -o console_output_style=count \ - tests + if [[ "$GITHUB_EVENT_NAME" = "release" ]]; then + TAG="${GITHUB_REF#refs/tags/v}" + else + TAG=$(cat esphome/const.py | sed -n -E "s/^__version__\s+=\s+\"(.+)\"$/\1/p") + today="$(date --utc '+%Y%m%d')" + TAG="${TAG}${today}" + fi + echo "::set-output name=tag::${TAG}" deploy-pypi: name: Build and publish to PyPi - if: github.repository == 'esphome/esphome' - needs: [lint-clang-format, lint-clang-tidy, lint-python, test, pytest] + if: github.repository == 'esphome/esphome' && github.event_name == 'release' runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -182,119 +53,85 @@ jobs: name: Build and publish docker containers if: github.repository == 'esphome/esphome' runs-on: ubuntu-latest - needs: [lint-clang-format, lint-clang-tidy, lint-python, test, pytest] + needs: [init] strategy: matrix: arch: [amd64, armv7, aarch64] - build_type: ["hassio", "docker"] + build_type: ["ha-addon", "docker"] steps: - - uses: actions/checkout@v2 - - name: Set TAG - run: | - TAG="${GITHUB_REF#refs/tags/v}" - echo "TAG=${TAG}" >> $GITHUB_ENV - - name: Set up env variables - run: | - base_version="3.4.0" + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' - if [[ "${{ matrix.build_type }}" == "hassio" ]]; then - build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}" - build_to="esphome/esphome-hassio-${{ matrix.arch }}" - dockerfile="docker/Dockerfile.hassio" - else - build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}" - build_to="esphome/esphome-${{ matrix.arch }}" - dockerfile="docker/Dockerfile" - fi + - name: Run build + run: | + docker/build.py \ + --tag "${{ needs.init.outputs.tag }}" \ + --arch "${{ matrix.arch }}" \ + --build-type "${{ matrix.build_type }}" \ + build - if [[ "${{ github.event.release.prerelease }}" == "true" ]]; then - cache_tag="beta" - else - cache_tag="latest" - fi + - name: Log in to docker hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Log in to the GitHub container registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - # Set env variables so these values don't need to be calculated again - echo "BUILD_FROM=${build_from}" >> $GITHUB_ENV - echo "BUILD_TO=${build_to}" >> $GITHUB_ENV - echo "DOCKERFILE=${dockerfile}" >> $GITHUB_ENV - echo "CACHE_TAG=${cache_tag}" >> $GITHUB_ENV - - name: Pull for cache - run: | - docker pull "${BUILD_TO}:${CACHE_TAG}" || true - - name: Register QEMU binfmt - run: docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset -p yes - - run: | - docker build \ - --build-arg "BUILD_FROM=${BUILD_FROM}" \ - --build-arg "BUILD_VERSION=${TAG}" \ - --tag "${BUILD_TO}:${TAG}" \ - --cache-from "${BUILD_TO}:${CACHE_TAG}" \ - --file "${DOCKERFILE}" \ - . - - name: Log in to docker hub - env: - DOCKER_USER: ${{ secrets.DOCKER_USER }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}" - - run: docker push "${BUILD_TO}:${TAG}" - - # Always publish to beta tag (also full releases) - - name: Publish docker beta tag - run: | - docker tag "${BUILD_TO}:${TAG}" "${BUILD_TO}:beta" - docker push "${BUILD_TO}:beta" - - - if: ${{ !github.event.release.prerelease }} - name: Publish docker latest tag - run: | - docker tag "${BUILD_TO}:${TAG}" "${BUILD_TO}:latest" - docker push "${BUILD_TO}:latest" + - name: Run push + run: | + docker/build.py \ + --tag "${{ needs.init.outputs.tag }}" \ + --arch "${{ matrix.arch }}" \ + --build-type "${{ matrix.build_type }}" \ + push deploy-docker-manifest: if: github.repository == 'esphome/esphome' runs-on: ubuntu-latest - needs: [deploy-docker] + needs: [init, deploy-docker] + strategy: + matrix: + build_type: ["ha-addon", "docker"] steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' - name: Enable experimental manifest support run: | mkdir -p ~/.docker echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json - - name: Set TAG - run: | - TAG="${GITHUB_REF#refs/tags/v}" - echo "TAG=${TAG}" >> $GITHUB_ENV + - name: Log in to docker hub - env: - DOCKER_USER: ${{ secrets.DOCKER_USER }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}" - - name: "Create the manifest" - run: | - docker manifest create esphome/esphome:${TAG} \ - esphome/esphome-aarch64:${TAG} \ - esphome/esphome-amd64:${TAG} \ - esphome/esphome-armv7:${TAG} - docker manifest push esphome/esphome:${TAG} + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Log in to the GitHub container registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Publish docker beta tag + - name: Run manifest run: | - docker manifest create esphome/esphome:beta \ - esphome/esphome-aarch64:${TAG} \ - esphome/esphome-amd64:${TAG} \ - esphome/esphome-armv7:${TAG} - docker manifest push esphome/esphome:beta - - - name: Publish docker latest tag - if: ${{ !github.event.release.prerelease }} - run: | - docker manifest create esphome/esphome:latest \ - esphome/esphome-aarch64:${TAG} \ - esphome/esphome-amd64:${TAG} \ - esphome/esphome-armv7:${TAG} - docker manifest push esphome/esphome:latest + docker/build.py \ + --tag "${{ needs.init.outputs.tag }}" \ + --build-type "${{ matrix.build_type }}" \ + manifest deploy-hassio-repo: - if: github.repository == 'esphome/esphome' + if: github.repository == 'esphome/esphome' && github.event_name == 'release' runs-on: ubuntu-latest needs: [deploy-docker] steps: diff --git a/docker/Dockerfile b/docker/Dockerfile index 0d126a2944..907c041119 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=esphome/esphome-base-amd64:3.4.0 +ARG BUILD_FROM=esphome/esphome-base:latest FROM ${BUILD_FROM} # First install requirements to leverage caching when requirements don't change diff --git a/docker/Dockerfile.hassio b/docker/Dockerfile.hassio index 5dd9339b18..ad80074ada 100644 --- a/docker/Dockerfile.hassio +++ b/docker/Dockerfile.hassio @@ -1,4 +1,4 @@ -ARG BUILD_FROM +ARG BUILD_FROM=esphome/esphome-hassio-base:latest FROM ${BUILD_FROM} # First install requirements to leverage caching when requirements don't change diff --git a/docker/Dockerfile.lint b/docker/Dockerfile.lint index 60d63152a0..3a090c3b41 100644 --- a/docker/Dockerfile.lint +++ b/docker/Dockerfile.lint @@ -1,4 +1,5 @@ -FROM esphome/esphome-lint-base:3.4.0 +ARG BUILD_FROM=esphome/esphome-lint-base:latest +FROM ${BUILD_FROM} COPY requirements.txt requirements_optional.txt requirements_test.txt docker/platformio_install_deps.py platformio.ini / RUN \ diff --git a/docker/build.py b/docker/build.py new file mode 100755 index 0000000000..97222df8d1 --- /dev/null +++ b/docker/build.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +from dataclasses import dataclass +import subprocess +import argparse +import platform +import shlex +import re +import sys + + +CHANNEL_DEV = 'dev' +CHANNEL_BETA = 'beta' +CHANNEL_RELEASE = 'release' +CHANNELS = [CHANNEL_DEV, CHANNEL_BETA, CHANNEL_RELEASE] + +ARCH_AMD64 = 'amd64' +ARCH_ARMV7 = 'armv7' +ARCH_AARCH64 = 'aarch64' +ARCHS = [ARCH_AMD64, ARCH_ARMV7, ARCH_AARCH64] + +TYPE_DOCKER = 'docker' +TYPE_HA_ADDON = 'ha-addon' +TYPE_LINT = 'lint' +TYPES = [TYPE_DOCKER, TYPE_HA_ADDON, TYPE_LINT] + + +BASE_VERSION = "3.6.0" + + +parser = argparse.ArgumentParser() +parser.add_argument("--tag", type=str, required=True, help="The main docker tag to push to. If a version number also adds latest and/or beta tag") +parser.add_argument("--arch", choices=ARCHS, required=False, help="The architecture to build for") +parser.add_argument("--build-type", choices=TYPES, required=True, help="The type of build to run") +parser.add_argument("--dry-run", action="store_true", help="Don't run any commands, just print them") +subparsers = parser.add_subparsers(help="Action to perform", dest="command", required=True) +build_parser = subparsers.add_parser("build", help="Build the image") +push_parser = subparsers.add_parser("push", help="Tag the already built image and push it to docker hub") +manifest_parser = subparsers.add_parser("manifest", help="Create a manifest from already pushed images") + + + +# only lists some possibilities, doesn't have to be perfect +# https://stackoverflow.com/a/45125525 +UNAME_TO_ARCH = { + "x86_64": ARCH_AMD64, + "aarch64": ARCH_AARCH64, + "aarch64_be": ARCH_AARCH64, + "arm": ARCH_ARMV7, +} + + +@dataclass(frozen=True) +class DockerParams: + build_from: str + build_to: str + manifest_to: str + dockerfile: str + + @classmethod + def for_type_arch(cls, build_type, arch): + prefix = { + TYPE_DOCKER: "esphome/esphome", + TYPE_HA_ADDON: "esphome/esphome-hassio", + TYPE_LINT: "esphome/esphome-lint" + }[build_type] + build_from = f"ghcr.io/{prefix}-base-{arch}:{BASE_VERSION}" + build_to = f"{prefix}-{arch}" + dockerfile = { + TYPE_DOCKER: "docker/Dockerfile", + TYPE_HA_ADDON: "docker/Dockerfile.hassio", + TYPE_LINT: "docker/Dockerfile.lint", + }[build_type] + return cls( + build_from=build_from, + build_to=build_to, + manifest_to=prefix, + dockerfile=dockerfile + ) + + +def main(): + args = parser.parse_args() + + def run_command(*cmd, ignore_error: bool = False): + print(f"$ {shlex.join(list(cmd))}") + if not args.dry_run: + rc = subprocess.call(list(cmd)) + if rc != 0 and not ignore_error: + print("Command failed") + sys.exit(1) + + # detect channel from tag + match = re.match(r'\d+\.\d+(?:\.\d+)?(b\d+)?', args.tag) + if match is None: + channel = CHANNEL_DEV + elif match.group(1) is None: + channel = CHANNEL_RELEASE + else: + channel = CHANNEL_BETA + + tags_to_push = [args.tag] + if channel == CHANNEL_DEV: + tags_to_push.append("dev") + elif channel == CHANNEL_BETA: + tags_to_push.append("beta") + elif channel == CHANNEL_RELEASE: + # Additionally push to beta + tags_to_push.append("beta") + tags_to_push.append("latest") + + if args.command == "build": + # 1. pull cache image + params = DockerParams.for_type_arch(args.build_type, args.arch) + cache_tag = { + CHANNEL_DEV: "dev", + CHANNEL_BETA: "beta", + CHANNEL_RELEASE: "latest", + }[channel] + cache_img = f"ghcr.io/{params.build_to}:{cache_tag}" + run_command("docker", "pull", cache_img, ignore_error=True) + + # 2. register QEMU binfmt (if not host arch) + is_native = UNAME_TO_ARCH.get(platform.machine()) == args.arch + if not is_native: + run_command( + "docker", "run", "--rm", "--privileged", "multiarch/qemu-user-static:5.2.0-2", + "--reset", "-p", "yes" + ) + + # 3. build + run_command( + "docker", "build", + "--build-arg", f"BUILD_FROM={params.build_from}", + "--build-arg", f"BUILD_VERSION={args.tag}", + "--tag", f"{params.build_to}:{args.tag}", + "--cache-from", cache_img, + "--file", params.dockerfile, + "." + ) + elif args.command == "push": + params = DockerParams.for_type_arch(args.build_type, args.arch) + imgs = [f"{params.build_to}:{tag}" for tag in tags_to_push] + imgs += [f"ghcr.io/{params.build_to}:{tag}" for tag in tags_to_push] + src = imgs[0] + # 1. tag images + for img in imgs[1:]: + run_command( + "docker", "tag", src, img + ) + # 2. push images + for img in imgs: + run_command( + "docker", "push", img + ) + elif args.command == "manifest": + manifest = DockerParams.for_type_arch(args.build_type, ARCH_AMD64).manifest_to + + targets = [f"{manifest}:{tag}" for tag in tags_to_push] + targets += [f"ghcr.io/{manifest}:{tag}" for tag in tags_to_push] + # 1. Create manifests + for target in targets: + cmd = ["docker", "manifest", "create", target] + for arch in ARCHS: + src = f"{DockerParams.for_type_arch(args.build_type, arch).build_to}:{args.tag}" + if target.startswith("ghcr.io"): + src = f"ghcr.io/{src}" + cmd.append(src) + run_command(*cmd) + # 2. Push manifests + for target in targets: + run_command( + "docker", "manifest", "push", target + ) + + +if __name__ == "__main__": + main() diff --git a/esphome/const.py b/esphome/const.py index 35f1e560e1..c336f38b22 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -1,10 +1,6 @@ """Constants used by esphome.""" -MAJOR_VERSION = 1 -MINOR_VERSION = 20 -PATCH_VERSION = "0b2" -__short_version__ = f"{MAJOR_VERSION}.{MINOR_VERSION}" -__version__ = f"{__short_version__}.{PATCH_VERSION}" +__version__ = "1.20.0b2" ESP_PLATFORM_ESP32 = "ESP32" ESP_PLATFORM_ESP8266 = "ESP8266" diff --git a/script/bump-docker-base-version.py b/script/bump-docker-base-version.py deleted file mode 100755 index f5f4399fd2..0000000000 --- a/script/bump-docker-base-version.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import re -import sys - - -def sub(path, pattern, repl, expected_count=1): - with open(path) as fh: - content = fh.read() - content, count = re.subn(pattern, repl, content, flags=re.MULTILINE) - if expected_count is not None: - assert count == expected_count, f"Pattern {pattern} replacement failed!" - with open(path, "wt") as fh: - fh.write(content) - - -def write_version(version: str): - for p in [ - ".github/workflows/ci-docker.yml", - ".github/workflows/release-dev.yml", - ".github/workflows/release.yml", - ]: - sub(p, r'base_version=".*"', f'base_version="{version}"') - - sub( - "docker/Dockerfile", - r"ARG BUILD_FROM=esphome/esphome-base-amd64:.*", - f"ARG BUILD_FROM=esphome/esphome-base-amd64:{version}", - ) - sub( - "docker/Dockerfile.lint", - r"FROM esphome/esphome-lint-base:.*", - f"FROM esphome/esphome-lint-base:{version}", - ) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("new_version", type=str) - args = parser.parse_args() - - version = args.new_version - print(f"Bumping to {version}") - write_version(version) - return 0 - - -if __name__ == "__main__": - sys.exit(main() or 0) diff --git a/script/bump-version.py b/script/bump-version.py index b7b048eb22..1f034344f9 100755 --- a/script/bump-version.py +++ b/script/bump-version.py @@ -50,16 +50,10 @@ def sub(path, pattern, repl, expected_count=1): def write_version(version: Version): - sub( - "esphome/const.py", r"^MAJOR_VERSION = \d+$", f"MAJOR_VERSION = {version.major}" - ) - sub( - "esphome/const.py", r"^MINOR_VERSION = \d+$", f"MINOR_VERSION = {version.minor}" - ) sub( "esphome/const.py", - r"^PATCH_VERSION = .*$", - f'PATCH_VERSION = "{version.full_patch}"', + r"^__version__ = .*$", + f'__version__ = "{version}"', )