From 272ceadbb00d5980aa00c0ad5043878a330d275f Mon Sep 17 00:00:00 2001 From: Otto Winter Date: Mon, 20 Sep 2021 09:07:38 +0200 Subject: [PATCH] Redo docker build system with buildkit+multi-stage and cache pio packages (#2338) --- .devcontainer/devcontainer.json | 3 +- .github/workflows/ci-docker.yml | 5 + .github/workflows/ci.yml | 152 +++++++++--------- .github/workflows/docker-lint-build.yml | 100 ------------ .github/workflows/release.yml | 20 ++- docker/Dockerfile | 135 ++++++++++++++-- docker/Dockerfile.dev | 1 - docker/Dockerfile.hassio | 25 --- docker/Dockerfile.lint | 10 -- docker/build.py | 94 +++++------ docker/docker_entrypoint.sh | 18 +++ .../etc/cont-init.d/10-requirements.sh | 0 .../etc/cont-init.d/20-nginx.sh | 0 .../hassio-rootfs/etc/cont-init.d/30-dirs.sh | 9 ++ .../etc/nginx/includes/mime.types | 0 .../etc/nginx/includes/proxy_params.conf | 0 .../etc/nginx/includes/server_params.conf | 0 .../etc/nginx/includes/ssl_params.conf | 0 .../etc/nginx/nginx.conf | 0 .../etc/nginx/servers/direct-ssl.disabled | 0 .../etc/nginx/servers/direct.disabled | 0 .../etc/nginx/servers/ingress.conf | 0 .../etc/services.d/esphome/finish | 0 .../etc/services.d/esphome/run | 3 + .../etc/services.d/nginx/finish | 0 .../etc/services.d/nginx/run | 0 esphome/platformio_api.py | 4 +- script/ci-custom.py | 4 +- script/devcontainer-post-create | 5 +- 29 files changed, 295 insertions(+), 293 deletions(-) delete mode 100644 .github/workflows/docker-lint-build.yml delete mode 100644 docker/Dockerfile.dev delete mode 100644 docker/Dockerfile.hassio delete mode 100644 docker/Dockerfile.lint create mode 100755 docker/docker_entrypoint.sh rename docker/{rootfs => hassio-rootfs}/etc/cont-init.d/10-requirements.sh (100%) rename docker/{rootfs => hassio-rootfs}/etc/cont-init.d/20-nginx.sh (100%) create mode 100644 docker/hassio-rootfs/etc/cont-init.d/30-dirs.sh rename docker/{rootfs => hassio-rootfs}/etc/nginx/includes/mime.types (100%) rename docker/{rootfs => hassio-rootfs}/etc/nginx/includes/proxy_params.conf (100%) rename docker/{rootfs => hassio-rootfs}/etc/nginx/includes/server_params.conf (100%) rename docker/{rootfs => hassio-rootfs}/etc/nginx/includes/ssl_params.conf (100%) rename docker/{rootfs => hassio-rootfs}/etc/nginx/nginx.conf (100%) rename docker/{rootfs => hassio-rootfs}/etc/nginx/servers/direct-ssl.disabled (100%) rename docker/{rootfs => hassio-rootfs}/etc/nginx/servers/direct.disabled (100%) rename docker/{rootfs => hassio-rootfs}/etc/nginx/servers/ingress.conf (100%) rename docker/{rootfs => hassio-rootfs}/etc/services.d/esphome/finish (100%) rename docker/{rootfs => hassio-rootfs}/etc/services.d/esphome/run (89%) rename docker/{rootfs => hassio-rootfs}/etc/services.d/nginx/finish (100%) rename docker/{rootfs => hassio-rootfs}/etc/services.d/nginx/run (100%) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3904962d7c..433e5d2792 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,6 @@ { "name": "ESPHome Dev", - "context": "..", - "dockerFile": "../docker/Dockerfile.dev", + "image": "esphome/esphome-lint:dev", "postCreateCommand": [ "script/devcontainer-post-create" ], diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index 45fd3e141b..33b15cb1dc 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -27,6 +27,11 @@ jobs: uses: actions/setup-python@v2 with: python-version: '3.9' + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set TAG run: | echo "TAG=check" >> $GITHUB_ENV diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b00e1b3835..875db0f9fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,61 +9,7 @@ on: pull_request: jobs: - ci-with-container: - name: ${{ matrix.name }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - id: clang-format - name: Run script/clang-format - - id: clang-tidy - name: Run script/clang-tidy for ESP8266 - options: --environment esp8266-tidy --grep ARDUINO_ARCH_ESP8266 - - id: clang-tidy - name: Run script/clang-tidy for ESP32 1/4 - options: --environment esp32-tidy --split-num 4 --split-at 1 - - id: clang-tidy - name: Run script/clang-tidy for ESP32 2/4 - options: --environment esp32-tidy --split-num 4 --split-at 2 - - id: clang-tidy - name: Run script/clang-tidy for ESP32 3/4 - options: --environment esp32-tidy --split-num 4 --split-at 3 - - id: clang-tidy - name: Run script/clang-tidy for ESP32 4/4 - options: --environment esp32-tidy --split-num 4 --split-at 4 - - # cpp lint job runs with esphome-lint docker image so that clang-format-* - # doesn't have to be installed - container: ghcr.io/esphome/esphome-lint:1.2 - steps: - - uses: actions/checkout@v2 - - - name: Register problem matchers - run: | - echo "::add-matcher::.github/workflows/matchers/clang-tidy.json" - echo "::add-matcher::.github/workflows/matchers/gcc.json" - - # Also run git-diff-index so that the step is marked as failed on formatting errors, - # since clang-format doesn't do anything but change files if -i is passed. - - name: Run clang-format - run: | - script/clang-format -i - git diff-index --quiet HEAD -- - if: ${{ matrix.id == 'clang-format' }} - - - name: Run clang-tidy - run: script/clang-tidy --all-headers --fix ${{ matrix.options }} - if: ${{ matrix.id == 'clang-tidy' }} - - - name: Suggested changes - run: script/ci-suggest-changes - if: always() - ci: - # Don't use the esphome-lint docker image because it may contain outdated requirements. - # This way, all dependencies are cached via the cache action. name: ${{ matrix.name }} runs-on: ubuntu-latest strategy: @@ -77,48 +23,85 @@ jobs: - id: test file: tests/test1.yaml name: Test tests/test1.yaml + pio_cache_key: test1 - id: test file: tests/test2.yaml name: Test tests/test2.yaml + pio_cache_key: test2 - id: test file: tests/test3.yaml name: Test tests/test3.yaml + pio_cache_key: test1 - id: test file: tests/test4.yaml name: Test tests/test4.yaml + pio_cache_key: test4 - id: test file: tests/test5.yaml name: Test tests/test5.yaml + pio_cache_key: test5 - id: pytest name: Run pytest + - id: clang-format + name: Run script/clang-format + - id: clang-tidy + name: Run script/clang-tidy for ESP8266 + options: --environment esp8266-tidy --grep ARDUINO_ARCH_ESP8266 + pio_cache_key: tidyesp8266 + - id: clang-tidy + name: Run script/clang-tidy for ESP32 1/4 + options: --environment esp32-tidy --split-num 4 --split-at 1 + pio_cache_key: tidyesp32 + - id: clang-tidy + name: Run script/clang-tidy for ESP32 2/4 + options: --environment esp32-tidy --split-num 4 --split-at 2 + pio_cache_key: tidyesp32 + - id: clang-tidy + name: Run script/clang-tidy for ESP32 3/4 + options: --environment esp32-tidy --split-num 4 --split-at 3 + pio_cache_key: tidyesp32 + - id: clang-tidy + name: Run script/clang-tidy for ESP32 4/4 + options: --environment esp32-tidy --split-num 4 --split-at 4 + pio_cache_key: tidyesp32 steps: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 + id: python with: python-version: '3.7' - name: Cache pip modules - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: ~/.cache/pip - key: esphome-pip-3.7-${{ hashFiles('setup.py') }} + key: pip-${{ steps.python.outputs.python-version }}-${{ hashFiles('requirements*.txt') }} restore-keys: | - esphome-pip-3.7- - - # Use per test platformio cache because tests have different platform versions - - name: Cache ~/.platformio - uses: actions/cache@v1 - with: - path: ~/.platformio - key: test-home-platformio-${{ matrix.file }}-${{ hashFiles('esphome/core/config.py') }} - restore-keys: | - test-home-platformio-${{ matrix.file }}- - if: ${{ matrix.id == 'test' }} + pip-${{ steps.python.outputs.python-version }}- - name: Set up python environment - run: script/setup + run: | + pip3 install -r requirements.txt -r requirements_optional.txt -r requirements_test.txt + pip3 install -e . + + # Use per check platformio cache because checks use different parts + - name: Cache platformio + uses: actions/cache@v2 + with: + path: ~/.platformio + key: platformio-${{ matrix.pio_cache_key }}-${{ hashFiles('platformio.ini') }} + restore-keys: | + platformio-${{ matrix.pio_cache_key }}- + if: matrix.id == 'test' || matrix.id == 'clang-tidy' + + - name: Install clang tools + run: | + sudo apt-get install \ + clang-format-11 \ + clang-tidy-11 + if: matrix.id == 'clang-tidy' || matrix.id == 'clang-format' - name: Register problem matchers run: | @@ -127,20 +110,45 @@ jobs: echo "::add-matcher::.github/workflows/matchers/python.json" echo "::add-matcher::.github/workflows/matchers/pytest.json" echo "::add-matcher::.github/workflows/matchers/gcc.json" + echo "::add-matcher::.github/workflows/matchers/clang-tidy.json" - name: Lint Custom run: | script/ci-custom.py script/build_codeowners.py --check - if: ${{ matrix.id == 'ci-custom' }} + if: matrix.id == 'ci-custom' + - name: Lint Python run: script/lint-python - if: ${{ matrix.id == 'lint-python' }} + if: matrix.id == 'lint-python' - run: esphome compile ${{ matrix.file }} - if: ${{ matrix.id == 'test' }} + if: matrix.id == 'test' + env: + # Also cache libdeps, store them in a ~/.platformio subfolder + PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps - name: Run pytest run: | pytest -vv --tb=native tests - if: ${{ matrix.id == 'pytest' }} + if: matrix.id == 'pytest' + + # Also run git-diff-index so that the step is marked as failed on formatting errors, + # since clang-format doesn't do anything but change files if -i is passed. + - name: Run clang-format + run: | + script/clang-format -i + git diff-index --quiet HEAD -- + if: matrix.id == 'clang-format' + + - name: Run clang-tidy + run: | + script/clang-tidy --all-headers --fix ${{ matrix.options }} + if: matrix.id == 'clang-tidy' + env: + # Also cache libdeps, store them in a ~/.platformio subfolder + PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps + + - name: Suggested changes + run: script/ci-suggest-changes + if: always() && (matrix.id == 'clang-tidy' || matrix.id == 'clang-format') diff --git a/.github/workflows/docker-lint-build.yml b/.github/workflows/docker-lint-build.yml deleted file mode 100644 index 8350d4c719..0000000000 --- a/.github/workflows/docker-lint-build.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: Build and publish lint docker image - -# Only run when docker paths change -on: - push: - branches: [dev] - paths: - - 'docker/Dockerfile.lint' - - 'requirements.txt' - - 'requirements_optional.txt' - - 'requirements_test.txt' - - 'platformio.ini' - - '.github/workflows/docker-lint-build.yml' - -jobs: - deploy-docker: - name: Build and publish docker containers - if: github.repository == 'esphome/esphome' - runs-on: ubuntu-latest - strategy: - matrix: - arch: [amd64, armv7, aarch64] - build_type: ["lint"] - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.9' - - name: Set TAG - run: | - echo "TAG=1.2" >> $GITHUB_ENV - - - name: Run build - run: | - docker/build.py \ - --tag "${TAG}" \ - --arch "${{ matrix.arch }}" \ - --build-type "${{ matrix.build_type }}" \ - build - - - name: Log in to docker hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USER }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Log in to the GitHub container registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Run push - run: | - docker/build.py \ - --tag "${TAG}" \ - --arch "${{ matrix.arch }}" \ - --build-type "${{ matrix.build_type }}" \ - push - - deploy-docker-manifest: - if: github.repository == 'esphome/esphome' - runs-on: ubuntu-latest - needs: [deploy-docker] - strategy: - matrix: - build_type: ["lint"] - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.9' - - name: Set TAG - run: | - echo "TAG=1.2" >> $GITHUB_ENV - - name: Enable experimental manifest support - run: | - mkdir -p ~/.docker - echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json - - - name: Log in to docker hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USER }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Log in to the GitHub container registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Run manifest - run: | - docker/build.py \ - --tag "${TAG}" \ - --build-type "${{ matrix.build_type }}" \ - manifest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3dc4952422..afd893d065 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,7 +57,7 @@ jobs: strategy: matrix: arch: [amd64, armv7, aarch64] - build_type: ["ha-addon", "docker"] + build_type: ["ha-addon", "docker", "lint"] steps: - uses: actions/checkout@v2 - name: Set up Python @@ -65,13 +65,10 @@ jobs: with: python-version: '3.9' - - name: Run build - run: | - docker/build.py \ - --tag "${{ needs.init.outputs.tag }}" \ - --arch "${{ matrix.arch }}" \ - --build-type "${{ matrix.build_type }}" \ - build + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 - name: Log in to docker hub uses: docker/login-action@v1 @@ -85,13 +82,14 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Run push + - name: Build and push run: | docker/build.py \ --tag "${{ needs.init.outputs.tag }}" \ --arch "${{ matrix.arch }}" \ --build-type "${{ matrix.build_type }}" \ - push + build \ + --push deploy-docker-manifest: if: github.repository == 'esphome/esphome' @@ -99,7 +97,7 @@ jobs: needs: [init, deploy-docker] strategy: matrix: - build_type: ["ha-addon", "docker"] + build_type: ["ha-addon", "docker", "lint"] steps: - uses: actions/checkout@v2 - name: Set up Python diff --git a/docker/Dockerfile b/docker/Dockerfile index 907c041119..d3ee219de8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,55 @@ -ARG BUILD_FROM=esphome/esphome-base:latest -FROM ${BUILD_FROM} +# Build these with the build.py script +# Example: +# python3 docker/build.py --tag dev --arch amd64 --build-type docker build + +# One of "docker", "hassio" +ARG BASEIMGTYPE=docker + +FROM ghcr.io/hassio-addons/debian-base/amd64:5.0.0 AS base-hassio-amd64 +FROM ghcr.io/hassio-addons/debian-base/aarch64:5.0.0 AS base-hassio-arm64 +FROM ghcr.io/hassio-addons/debian-base/armv7:5.0.0 AS base-hassio-armv7 +FROM debian:bullseye-20210816-slim AS base-docker-amd64 +FROM debian:bullseye-20210816-slim AS base-docker-arm64 +FROM debian:bullseye-20210816-slim AS base-docker-armv7 + +# Use TARGETARCH/TARGETVARIANT defined by docker +# https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope +FROM base-${BASEIMGTYPE}-${TARGETARCH}${TARGETVARIANT} AS base + +RUN \ + apt-get update \ + # Use pinned versions so that we get updates with build caching + && apt-get install -y --no-install-recommends \ + python3=3.9.2-3 \ + python3-pip=20.3.4-4 \ + python3-setuptools=52.0.0-4 \ + python3-pil=8.1.2+dfsg-0.3 \ + python3-cryptography=3.3.2-1 \ + iputils-ping=3:20210202-1 \ + git=1:2.30.2-1 \ + curl=7.74.0-1.3+b1 \ + && rm -rf \ + /tmp/* \ + /var/{cache,log}/* \ + /var/lib/apt/lists/* + +ENV \ + # Fix click python3 lang warning https://click.palletsprojects.com/en/7.x/python3/ + LANG=C.UTF-8 LC_ALL=C.UTF-8 \ + # Store globally installed pio libs in /piolibs + PLATFORMIO_GLOBALLIB_DIR=/piolibs + +RUN \ + # Ubuntu python3-pip is missing wheel + pip3 install --no-cache-dir \ + wheel==0.36.2 \ + platformio==5.2.0 \ + # Change some platformio settings + && platformio settings set enable_telemetry No \ + && platformio settings set check_libraries_interval 1000000 \ + && platformio settings set check_platformio_interval 1000000 \ + && platformio settings set check_platforms_interval 1000000 \ + && mkdir -p /piolibs # First install requirements to leverage caching when requirements don't change COPY requirements.txt requirements_optional.txt docker/platformio_install_deps.py platformio.ini / @@ -7,9 +57,14 @@ RUN \ pip3 install --no-cache-dir -r /requirements.txt -r /requirements_optional.txt \ && /platformio_install_deps.py /platformio.ini -# Then copy esphome and install -COPY . . -RUN pip3 install --no-cache-dir -e . + + +# ======================= docker-type image ======================= +FROM base AS docker + +# Copy esphome and install +COPY . /esphome +RUN pip3 install --no-cache-dir -e /esphome # Settings for dashboard ENV USERNAME="" PASSWORD="" @@ -17,14 +72,74 @@ ENV USERNAME="" PASSWORD="" # Expose the dashboard to Docker EXPOSE 6052 -# Run healthcheck (heartbeat) -HEALTHCHECK --interval=30s --timeout=30s \ - CMD curl --fail http://localhost:6052 || exit 1 +COPY docker/docker_entrypoint.sh /entrypoint.sh # The directory the user should mount their configuration files to +VOLUME /config WORKDIR /config -# Set entrypoint to esphome so that the user doesn't have to type 'esphome' +# Set entrypoint to esphome (via a script) so that the user doesn't have to type 'esphome' # in every docker command twice -ENTRYPOINT ["esphome"] +ENTRYPOINT ["/entrypoint.sh"] # When no arguments given, start the dashboard in the workdir CMD ["dashboard", "/config"] + + + + +# ======================= hassio-type image ======================= +FROM base AS hassio + +RUN \ + apt-get update \ + # Use pinned versions so that we get updates with build caching + && apt-get install -y --no-install-recommends \ + nginx=1.18.0-6.1 \ + && rm -rf \ + /tmp/* \ + /var/{cache,log}/* \ + /var/lib/apt/lists/* + +ARG BUILD_VERSION=dev + +# Copy root filesystem +COPY docker/hassio-rootfs/ / + +# Copy esphome and install +COPY . /esphome +RUN pip3 install --no-cache-dir -e /esphome + +# Labels +LABEL \ + io.hass.name="ESPHome" \ + io.hass.description="Manage and program ESP8266/ESP32 microcontrollers through YAML configuration files" \ + io.hass.type="addon" \ + io.hass.version="${BUILD_VERSION}" + # io.hass.arch is inherited from addon-debian-base + + + + +# ======================= lint-type image ======================= +FROM base AS lint + +ENV \ + PLATFORMIO_CORE_DIR=/esphome/.temp/platformio + +RUN \ + apt-get update \ + # Use pinned versions so that we get updates with build caching + && apt-get install -y --no-install-recommends \ + clang-format-11=1:11.0.1-2 \ + clang-tidy-11=1:11.0.1-2 \ + patch=2.7.6-7 \ + software-properties-common=0.96.20.2-2.1 \ + nano=5.4-2 \ + build-essential=12.9 \ + python3-dev=3.9.2-3 \ + && rm -rf \ + /tmp/* \ + /var/{cache,log}/* \ + /var/lib/apt/lists/* + +VOLUME ["/esphome"] +WORKDIR /esphome diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev deleted file mode 100644 index c646ce989b..0000000000 --- a/docker/Dockerfile.dev +++ /dev/null @@ -1 +0,0 @@ -FROM esphome/esphome-lint:1.2 diff --git a/docker/Dockerfile.hassio b/docker/Dockerfile.hassio deleted file mode 100644 index ad80074ada..0000000000 --- a/docker/Dockerfile.hassio +++ /dev/null @@ -1,25 +0,0 @@ -ARG BUILD_FROM=esphome/esphome-hassio-base:latest -FROM ${BUILD_FROM} - -# First install requirements to leverage caching when requirements don't change -COPY requirements.txt requirements_optional.txt docker/platformio_install_deps.py platformio.ini / -RUN \ - pip3 install --no-cache-dir -r /requirements.txt -r /requirements_optional.txt \ - && /platformio_install_deps.py /platformio.ini - -# Copy root filesystem -COPY docker/rootfs/ / - -# Then copy esphome and install -COPY . /opt/esphome/ -RUN pip3 install --no-cache-dir -e /opt/esphome - -# Build arguments -ARG BUILD_VERSION=dev - -# Labels -LABEL \ - io.hass.name="ESPHome" \ - io.hass.description="Manage and program ESP8266/ESP32 microcontrollers through YAML configuration files" \ - io.hass.type="addon" \ - io.hass.version=${BUILD_VERSION} diff --git a/docker/Dockerfile.lint b/docker/Dockerfile.lint deleted file mode 100644 index 3a090c3b41..0000000000 --- a/docker/Dockerfile.lint +++ /dev/null @@ -1,10 +0,0 @@ -ARG BUILD_FROM=esphome/esphome-lint-base:latest -FROM ${BUILD_FROM} - -COPY requirements.txt requirements_optional.txt requirements_test.txt docker/platformio_install_deps.py platformio.ini / -RUN \ - pip3 install --no-cache-dir -r /requirements.txt -r /requirements_optional.txt -r /requirements_test.txt \ - && /platformio_install_deps.py /platformio.ini - -VOLUME ["/esphome"] -WORKDIR /esphome diff --git a/docker/build.py b/docker/build.py index c926b3653b..1904457989 100755 --- a/docker/build.py +++ b/docker/build.py @@ -2,7 +2,7 @@ from dataclasses import dataclass import subprocess import argparse -import platform +from platform import machine import shlex import re import sys @@ -24,9 +24,6 @@ TYPE_LINT = 'lint' TYPES = [TYPE_DOCKER, TYPE_HA_ADDON, TYPE_LINT] -BASE_VERSION = "4.2.0" - - parser = argparse.ArgumentParser() parser.add_argument("--tag", type=str, required=True, help="The main docker tag to push to. If a version number also adds latest and/or beta tag") parser.add_argument("--arch", choices=ARCHS, required=False, help="The architecture to build for") @@ -34,27 +31,17 @@ parser.add_argument("--build-type", choices=TYPES, required=True, help="The type parser.add_argument("--dry-run", action="store_true", help="Don't run any commands, just print them") subparsers = parser.add_subparsers(help="Action to perform", dest="command", required=True) build_parser = subparsers.add_parser("build", help="Build the image") -push_parser = subparsers.add_parser("push", help="Tag the already built image and push it to docker hub") +build_parser.add_argument("--push", help="Also push the images") manifest_parser = subparsers.add_parser("manifest", help="Create a manifest from already pushed images") - -# only lists some possibilities, doesn't have to be perfect -# https://stackoverflow.com/a/45125525 -UNAME_TO_ARCH = { - "x86_64": ARCH_AMD64, - "aarch64": ARCH_AARCH64, - "aarch64_be": ARCH_AARCH64, - "arm": ARCH_ARMV7, -} - - @dataclass(frozen=True) class DockerParams: - build_from: str build_to: str manifest_to: str - dockerfile: str + baseimgtype: str + platform: str + target: str @classmethod def for_type_arch(cls, build_type, arch): @@ -63,18 +50,28 @@ class DockerParams: TYPE_HA_ADDON: "esphome/esphome-hassio", TYPE_LINT: "esphome/esphome-lint" }[build_type] - build_from = f"ghcr.io/{prefix}-base-{arch}:{BASE_VERSION}" build_to = f"{prefix}-{arch}" - dockerfile = { - TYPE_DOCKER: "docker/Dockerfile", - TYPE_HA_ADDON: "docker/Dockerfile.hassio", - TYPE_LINT: "docker/Dockerfile.lint", + baseimgtype = { + TYPE_DOCKER: "docker", + TYPE_HA_ADDON: "hassio", + TYPE_LINT: "docker", + }[build_type] + platform = { + ARCH_AMD64: "linux/amd64", + ARCH_ARMV7: "linux/arm/v7", + ARCH_AARCH64: "linux/arm64", + }[arch] + target = { + TYPE_DOCKER: "docker", + TYPE_HA_ADDON: "hassio", + TYPE_LINT: "lint", }[build_type] return cls( - build_from=build_from, build_to=build_to, manifest_to=prefix, - dockerfile=dockerfile + baseimgtype=baseimgtype, + platform=platform, + target=target, ) @@ -117,41 +114,26 @@ def main(): CHANNEL_RELEASE: "latest", }[channel] cache_img = f"ghcr.io/{params.build_to}:{cache_tag}" - run_command("docker", "pull", cache_img, ignore_error=True) - # 2. register QEMU binfmt (if not host arch) - is_native = UNAME_TO_ARCH.get(platform.machine()) == args.arch - if not is_native: - run_command( - "docker", "run", "--rm", "--privileged", "multiarch/qemu-user-static:5.2.0-2", - "--reset", "-p", "yes" - ) - - # 3. build - run_command( - "docker", "build", - "--build-arg", f"BUILD_FROM={params.build_from}", - "--build-arg", f"BUILD_VERSION={args.tag}", - "--tag", f"{params.build_to}:{args.tag}", - "--cache-from", cache_img, - "--file", params.dockerfile, - "." - ) - elif args.command == "push": - params = DockerParams.for_type_arch(args.build_type, args.arch) imgs = [f"{params.build_to}:{tag}" for tag in tags_to_push] imgs += [f"ghcr.io/{params.build_to}:{tag}" for tag in tags_to_push] - src = imgs[0] - # 1. tag images - for img in imgs[1:]: - run_command( - "docker", "tag", src, img - ) - # 2. push images + + # 3. build + cmd = [ + "docker", "buildx", "build", + "--build-arg", f"BASEIMGTYPE={params.baseimgtype}", + "--build-arg", f"BUILD_VERSION={args.tag}", + "--cache-from", cache_img, + "--file", "docker/Dockerfile", + "--platform", params.platform, + "--target", params.target, + ] for img in imgs: - run_command( - "docker", "push", img - ) + cmd += ["--tag", img] + if args.push: + cmd.append("--push") + + run_command(*cmd, ".") elif args.command == "manifest": manifest = DockerParams.for_type_arch(args.build_type, ARCH_AMD64).manifest_to diff --git a/docker/docker_entrypoint.sh b/docker/docker_entrypoint.sh new file mode 100755 index 0000000000..b3905d1fed --- /dev/null +++ b/docker/docker_entrypoint.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# If /cache is mounted, use that as PIO's coredir +# otherwise use path in /config (so that PIO packages aren't downloaded on each compile) + +if [[ -d /cache ]]; then + export PLATFORMIO_CORE_DIR=/cache/platformio +else + export PLATFORMIO_CORE_DIR=/config/.esphome/platformio +fi + +if [[ ! -d "${PLATFORMIO_CORE_DIR}" ]]; then + echo "Creating cache directory ${PLATFORMIO_CORE_DIR}" + echo "You can change this behavior by mounting a directory to the container's /cache directory." + mkdir -p "${PLATFORMIO_CORE_DIR}" +fi + +exec esphome "$@" diff --git a/docker/rootfs/etc/cont-init.d/10-requirements.sh b/docker/hassio-rootfs/etc/cont-init.d/10-requirements.sh similarity index 100% rename from docker/rootfs/etc/cont-init.d/10-requirements.sh rename to docker/hassio-rootfs/etc/cont-init.d/10-requirements.sh diff --git a/docker/rootfs/etc/cont-init.d/20-nginx.sh b/docker/hassio-rootfs/etc/cont-init.d/20-nginx.sh similarity index 100% rename from docker/rootfs/etc/cont-init.d/20-nginx.sh rename to docker/hassio-rootfs/etc/cont-init.d/20-nginx.sh diff --git a/docker/hassio-rootfs/etc/cont-init.d/30-dirs.sh b/docker/hassio-rootfs/etc/cont-init.d/30-dirs.sh new file mode 100644 index 0000000000..301fe4db63 --- /dev/null +++ b/docker/hassio-rootfs/etc/cont-init.d/30-dirs.sh @@ -0,0 +1,9 @@ +#!/usr/bin/with-contenv bashio +# ============================================================================== +# Community Hass.io Add-ons: ESPHome +# This files creates all directories used by esphome +# ============================================================================== + +PLATFORMIO_CORE_DIR=/data/cache/platformio + +mkdir -p "${PLATFORMIO_CORE_DIR}" diff --git a/docker/rootfs/etc/nginx/includes/mime.types b/docker/hassio-rootfs/etc/nginx/includes/mime.types similarity index 100% rename from docker/rootfs/etc/nginx/includes/mime.types rename to docker/hassio-rootfs/etc/nginx/includes/mime.types diff --git a/docker/rootfs/etc/nginx/includes/proxy_params.conf b/docker/hassio-rootfs/etc/nginx/includes/proxy_params.conf similarity index 100% rename from docker/rootfs/etc/nginx/includes/proxy_params.conf rename to docker/hassio-rootfs/etc/nginx/includes/proxy_params.conf diff --git a/docker/rootfs/etc/nginx/includes/server_params.conf b/docker/hassio-rootfs/etc/nginx/includes/server_params.conf similarity index 100% rename from docker/rootfs/etc/nginx/includes/server_params.conf rename to docker/hassio-rootfs/etc/nginx/includes/server_params.conf diff --git a/docker/rootfs/etc/nginx/includes/ssl_params.conf b/docker/hassio-rootfs/etc/nginx/includes/ssl_params.conf similarity index 100% rename from docker/rootfs/etc/nginx/includes/ssl_params.conf rename to docker/hassio-rootfs/etc/nginx/includes/ssl_params.conf diff --git a/docker/rootfs/etc/nginx/nginx.conf b/docker/hassio-rootfs/etc/nginx/nginx.conf similarity index 100% rename from docker/rootfs/etc/nginx/nginx.conf rename to docker/hassio-rootfs/etc/nginx/nginx.conf diff --git a/docker/rootfs/etc/nginx/servers/direct-ssl.disabled b/docker/hassio-rootfs/etc/nginx/servers/direct-ssl.disabled similarity index 100% rename from docker/rootfs/etc/nginx/servers/direct-ssl.disabled rename to docker/hassio-rootfs/etc/nginx/servers/direct-ssl.disabled diff --git a/docker/rootfs/etc/nginx/servers/direct.disabled b/docker/hassio-rootfs/etc/nginx/servers/direct.disabled similarity index 100% rename from docker/rootfs/etc/nginx/servers/direct.disabled rename to docker/hassio-rootfs/etc/nginx/servers/direct.disabled diff --git a/docker/rootfs/etc/nginx/servers/ingress.conf b/docker/hassio-rootfs/etc/nginx/servers/ingress.conf similarity index 100% rename from docker/rootfs/etc/nginx/servers/ingress.conf rename to docker/hassio-rootfs/etc/nginx/servers/ingress.conf diff --git a/docker/rootfs/etc/services.d/esphome/finish b/docker/hassio-rootfs/etc/services.d/esphome/finish similarity index 100% rename from docker/rootfs/etc/services.d/esphome/finish rename to docker/hassio-rootfs/etc/services.d/esphome/finish diff --git a/docker/rootfs/etc/services.d/esphome/run b/docker/hassio-rootfs/etc/services.d/esphome/run similarity index 89% rename from docker/rootfs/etc/services.d/esphome/run rename to docker/hassio-rootfs/etc/services.d/esphome/run index f806c50929..6218b200bd 100755 --- a/docker/rootfs/etc/services.d/esphome/run +++ b/docker/hassio-rootfs/etc/services.d/esphome/run @@ -22,5 +22,8 @@ if bashio::config.has_value 'relative_url'; then export ESPHOME_DASHBOARD_RELATIVE_URL=$(bashio::config 'relative_url') fi +export PLATFORMIO_CORE_DIR=/data/cache/platformio +export PLATFORMIO_GLOBALLIB_DIR=/piolibs + bashio::log.info "Starting ESPHome dashboard..." exec esphome dashboard /config/esphome --socket /var/run/esphome.sock --hassio diff --git a/docker/rootfs/etc/services.d/nginx/finish b/docker/hassio-rootfs/etc/services.d/nginx/finish similarity index 100% rename from docker/rootfs/etc/services.d/nginx/finish rename to docker/hassio-rootfs/etc/services.d/nginx/finish diff --git a/docker/rootfs/etc/services.d/nginx/run b/docker/hassio-rootfs/etc/services.d/nginx/run similarity index 100% rename from docker/rootfs/etc/services.d/nginx/run rename to docker/hassio-rootfs/etc/services.d/nginx/run diff --git a/esphome/platformio_api.py b/esphome/platformio_api.py index 6506a44e88..a99081a650 100644 --- a/esphome/platformio_api.py +++ b/esphome/platformio_api.py @@ -67,8 +67,8 @@ FILTER_PLATFORMIO_LINES = [ def run_platformio_cli(*args, **kwargs) -> Union[str, int]: os.environ["PLATFORMIO_FORCE_COLOR"] = "true" os.environ["PLATFORMIO_BUILD_DIR"] = os.path.abspath(CORE.relative_pioenvs_path()) - os.environ["PLATFORMIO_LIBDEPS_DIR"] = os.path.abspath( - CORE.relative_piolibdeps_path() + os.environ.setdefault( + "PLATFORMIO_LIBDEPS_DIR", os.path.abspath(CORE.relative_piolibdeps_path()) ) cmd = ["platformio"] + list(args) diff --git a/script/ci-custom.py b/script/ci-custom.py index cdc450a96b..3191842d8c 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -217,7 +217,9 @@ def lint_ext_check(fname): ) -@lint_file_check(exclude=["docker/rootfs/*", "docker/*.py", "script/*", "setup.py"]) +@lint_file_check( + exclude=["**.sh", "docker/hassio-rootfs/**", "docker/*.py", "script/*", "setup.py"] +) def lint_executable_bit(fname): ex = EXECUTABLE_BIT[fname] if ex != 100644: diff --git a/script/devcontainer-post-create b/script/devcontainer-post-create index 4b4dc5b6c9..120ab3307d 100755 --- a/script/devcontainer-post-create +++ b/script/devcontainer-post-create @@ -12,7 +12,6 @@ if [ ! -f $cpp_json ]; then pio init --ide vscode --silent sed -i "/\\/workspaces\/esphome\/include/d" $cpp_json else - echo "Cpp environment already configured. To reconfigure it you could run one the following commands:" - echo " pio init --ide vscode -e livingroom8266" - echo " pio init --ide vscode -e livingroom32" + echo "Cpp environment already configured. To reconfigure it you can run one the following commands:" + echo " pio init --ide vscode" fi