From 9bb06782b26b82b97395069253af9c501b3e2fa1 Mon Sep 17 00:00:00 2001 From: Otto Winter Date: Wed, 13 Jun 2018 11:13:53 +0200 Subject: [PATCH] Automate HassIO builds using Gitlab CI (#52) * Add Gitlab CI * Fix aarch64 edge build --- .dockerignore | 1 + .gitlab-ci.yml | 131 ++++++++++++++ docker/Dockerfile.aarch64 | 20 +++ docker/Dockerfile.amd64 | 19 ++ docker/Dockerfile.armhf | 20 +++ docker/Dockerfile.builder | 32 ++++ docker/Dockerfile.i386 | 19 ++ docker/Dockerfile.lint | 4 + docker/hassio-builder.sh | 318 ++++++++++++++++++++++++++++++++++ docker/platformio-esp8266.ini | 7 + docker/platformio.ini | 10 +- esphomeyaml-edge/build.json | 2 +- esphomeyaml/Dockerfile | 44 ----- esphomeyaml/build.json | 10 -- 14 files changed, 577 insertions(+), 60 deletions(-) create mode 100644 .gitlab-ci.yml create mode 100644 docker/Dockerfile.aarch64 create mode 100644 docker/Dockerfile.amd64 create mode 100644 docker/Dockerfile.armhf create mode 100644 docker/Dockerfile.builder create mode 100644 docker/Dockerfile.i386 create mode 100644 docker/Dockerfile.lint create mode 100755 docker/hassio-builder.sh create mode 100644 docker/platformio-esp8266.ini delete mode 100644 esphomeyaml/Dockerfile delete mode 100644 esphomeyaml/build.json diff --git a/.dockerignore b/.dockerignore index 57a62c809e..228d993833 100644 --- a/.dockerignore +++ b/.dockerignore @@ -106,3 +106,4 @@ venv.bak/ config/ examples/ Dockerfile +.git/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000..650f8f4969 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,131 @@ +--- +# Based on https://gitlab.com/hassio-addons/addon-node-red/blob/master/.gitlab-ci.yml +variables: + DOCKER_DRIVER: overlay2 + +stages: + - lint + - build + - deploy + +.lint: &lint + stage: lint + tags: + - python2.7 + - esphomeyaml-lint + +.hassio-builder: &hassio-builder + before_script: + - docker info + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" + services: + - docker:dind + tags: + - hassio-builder + +flake8: + <<: *lint + script: + - flake8 esphomeyaml + +pylint: + <<: *lint + script: + - pylint esphomeyaml + +.build: &build + <<: *hassio-builder + stage: build + script: + - | + hassio-builder.sh \ + -t . \ + -i ottowinter/esphomeyaml-hassio-${ADDON_ARCH} \ + -d "$CI_REGISTRY" \ + --${ADDON_ARCH} + - | + docker tag \ + "${CI_REGISTRY}/ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:dev" \ + "${CI_REGISTRY}/ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:${CI_COMMIT_SHA}" + - docker push "${CI_REGISTRY}/ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:${CI_COMMIT_SHA}" + - docker push "${CI_REGISTRY}/ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:dev" + +# Generic deploy template +.deploy: &deploy + <<: *hassio-builder + stage: deploy + script: + - version=${CI_COMMIT_TAG:1} + - echo "Publishing version ${version}" + - docker login -u "$DOCKER_USER" -p "$DOCKER_PASSWORD" + - docker pull "${CI_REGISTRY}/ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:${CI_COMMIT_SHA}" + - | + docker tag \ + "${CI_REGISTRY}/ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:${CI_COMMIT_SHA}" \ + "ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:${version}" + - | + docker tag \ + "ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:${version}" \ + "ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:latest" + - docker push "ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:${version}" + - docker push "ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:latest" + +# Build jobs +build:armhf: + <<: *build + variables: + ADDON_ARCH: armhf + +build:aarch64: + <<: *build + variables: + ADDON_ARCH: aarch64 + +build:i386: + <<: *build + variables: + ADDON_ARCH: i386 + +build:amd64: + <<: *build + variables: + ADDON_ARCH: amd64 + +# Deploy jobs +deploy:armhf: + <<: *deploy + variables: + ADDON_ARCH: armhf + only: + - /^v\d+\.\d+\.\d+(?:(?:(?:\+|\.)?[a-zA-Z0-9]+)*)?$/ + except: + - /^(?!master).+@/ + +deploy:aarch64: + <<: *deploy + variables: + ADDON_ARCH: aarch64 + only: + - /^v\d+\.\d+\.\d+(?:(?:(?:\+|\.)?[a-zA-Z0-9]+)*)?$/ + except: + - /^(?!master).+@/ + +deploy:i386: + <<: *deploy + variables: + ADDON_ARCH: i386 + only: + - /^v\d+\.\d+\.\d+(?:(?:(?:\+|\.)?[a-zA-Z0-9]+)*)?$/ + except: + - /^(?!master).+@/ + + +deploy:amd64: + <<: *deploy + variables: + ADDON_ARCH: amd64 + only: + - /^v\d+\.\d+\.\d+(?:(?:(?:\+|\.)?[a-zA-Z0-9]+)*)?$/ + except: + - /^(?!master).+@/ + diff --git a/docker/Dockerfile.aarch64 b/docker/Dockerfile.aarch64 new file mode 100644 index 0000000000..72bc408db2 --- /dev/null +++ b/docker/Dockerfile.aarch64 @@ -0,0 +1,20 @@ +# Dockerfile for aarch64 version of HassIO add-on +FROM homeassistant/aarch64-base:latest + +RUN apk add --no-cache \ + python2 \ + py2-pip \ + git \ + openssh \ + libc6-compat \ + && \ + pip install --no-cache-dir --no-binary :all: platformio && \ + platformio settings set enable_telemetry No + +COPY docker/platformio-esp8266.ini /pio/platformio.ini +RUN platformio run -d /pio; rm -rf /pio + +COPY . . +RUN pip install --no-cache-dir --no-binary :all: -e . + +CMD ["esphomeyaml", "/config/esphomeyaml", "dashboard"] diff --git a/docker/Dockerfile.amd64 b/docker/Dockerfile.amd64 new file mode 100644 index 0000000000..18a51151a5 --- /dev/null +++ b/docker/Dockerfile.amd64 @@ -0,0 +1,19 @@ +# Dockerfile for amd64 version of HassIO add-on +FROM ubuntu:bionic + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python \ + python-pip \ + python-setuptools \ + git \ + && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/*rm -rf /var/lib/apt/lists/* /tmp/* && \ + pip install --no-cache-dir --no-binary :all: platformio && \ + platformio settings set enable_telemetry No + +COPY docker/platformio.ini /pio/platformio.ini +RUN platformio run -d /pio; rm -rf /pio + +COPY . . +RUN pip install --no-cache-dir --no-binary :all: -e . + +CMD ["esphomeyaml", "/config/esphomeyaml", "dashboard"] diff --git a/docker/Dockerfile.armhf b/docker/Dockerfile.armhf new file mode 100644 index 0000000000..430e604202 --- /dev/null +++ b/docker/Dockerfile.armhf @@ -0,0 +1,20 @@ +# Dockerfile for armhf version of HassIO add-on +FROM homeassistant/armhf-base:latest + +RUN apk add --no-cache \ + python2 \ + py2-pip \ + git \ + openssh \ + libc6-compat \ + && \ + pip install --no-cache-dir --no-binary :all: platformio && \ + platformio settings set enable_telemetry No + +COPY docker/platformio-esp8266.ini /pio/platformio.ini +RUN platformio run -d /pio; rm -rf /pio + +COPY . . +RUN pip install --no-cache-dir --no-binary :all: -e . + +CMD ["esphomeyaml", "/config/esphomeyaml", "dashboard"] diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder new file mode 100644 index 0000000000..9fb7ad3604 --- /dev/null +++ b/docker/Dockerfile.builder @@ -0,0 +1,32 @@ +FROM multiarch/ubuntu-core:amd64-xenial + +# setup locals +RUN apt-get update && apt-get install -y \ + jq \ + git \ + python3-setuptools \ + && rm -rf /var/lib/apt/lists/* \ +ENV LANG C.UTF-8 + +# Install docker +# https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/ +RUN apt-get update && apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + software-properties-common \ + && rm -rf /var/lib/apt/lists/* \ + && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \ + && add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ + && apt-get update && apt-get install -y docker-ce \ + && rm -rf /var/lib/apt/lists/* + +# setup arm binary support +RUN apt-get update && apt-get install -y \ + qemu-user-static \ + binfmt-support \ + && rm -rf /var/lib/apt/lists/* + +COPY hassio-builder.sh /usr/bin/ + +WORKDIR /data diff --git a/docker/Dockerfile.i386 b/docker/Dockerfile.i386 new file mode 100644 index 0000000000..599a98fb04 --- /dev/null +++ b/docker/Dockerfile.i386 @@ -0,0 +1,19 @@ +# Dockerfile for i386 version of HassIO add-on +FROM i386/ubuntu:bionic + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python \ + python-pip \ + python-setuptools \ + git \ + && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/*rm -rf /var/lib/apt/lists/* /tmp/* && \ + pip install --no-cache-dir --no-binary :all: platformio && \ + platformio settings set enable_telemetry No + +COPY docker/platformio.ini /pio/platformio.ini +RUN platformio run -d /pio; rm -rf /pio + +COPY . . +RUN pip install --no-cache-dir --no-binary :all: -e . + +CMD ["esphomeyaml", "/config/esphomeyaml", "dashboard"] diff --git a/docker/Dockerfile.lint b/docker/Dockerfile.lint new file mode 100644 index 0000000000..6be70d8200 --- /dev/null +++ b/docker/Dockerfile.lint @@ -0,0 +1,4 @@ +FROM python:2.7 + +RUN pip install -r requirements.txt && \ + pip install flake8==3.5.0 pylint==1.8.4 diff --git a/docker/hassio-builder.sh b/docker/hassio-builder.sh new file mode 100755 index 0000000000..699e028748 --- /dev/null +++ b/docker/hassio-builder.sh @@ -0,0 +1,318 @@ +#!/usr/bin/env bash +# Based on Home Assistant's docker builder +###################### +# Hass.io Build-env +###################### +set -e + +echo -- "$@" + +#### Variable #### + +DOCKER_TIMEOUT=20 +DOCKER_PID=-1 +DOCKER_HUB="" +DOCKER_CACHE="true" +DOCKER_LOCAL="false" +TARGET="" +IMAGE="" +BUILD_LIST=() +BUILD_TASKS=() + +#### Misc functions #### + +function print_help() { + cat << EOF +Hass.io build-env for ecosystem: +docker run --rm homeassistant/{arch}-builder:latest [options] + +Options: + -h, --help + Display this help and exit. + + Repository / Data + -t, --target + Set local folder or path inside repository for build. + + Version/Image handling + -i, --image + Overwrite image name of build / support {arch} + + Architecture + --armhf + Build for arm. + --amd64 + Build for intel/amd 64bit. + --aarch64 + Build for arm 64bit. + --i386 + Build for intel/amd 32bit. + --all + Build all architecture. + + Build handling + --no-cache + Disable cache for the build (from latest). + -d, --docker-hub + Set or overwrite the docker repository. + + Use the host docker socket if mapped into container: + /var/run/docker.sock + +EOF + + exit 1 +} + +#### Docker functions #### + +function start_docker() { + local starttime + local endtime + + if [ -S "/var/run/docker.sock" ]; then + echo "[INFO] Use host docker setup with '/var/run/docker.sock'" + DOCKER_LOCAL="true" + return 0 + fi + + echo "[INFO] Starting docker." + dockerd 2> /dev/null & + DOCKER_PID=$! + + echo "[INFO] Waiting for docker to initialize..." + starttime="$(date +%s)" + endtime="$(date +%s)" + until docker info >/dev/null 2>&1; do + if [ $((endtime - starttime)) -le ${DOCKER_TIMEOUT} ]; then + sleep 1 + endtime=$(date +%s) + else + echo "[ERROR] Timeout while waiting for docker to come up" + exit 1 + fi + done + echo "[INFO] Docker was initialized" +} + + +function stop_docker() { + local starttime + local endtime + + if [ "$DOCKER_LOCAL" == "true" ]; then + return 0 + fi + + echo "[INFO] Stopping in container docker..." + if [ "$DOCKER_PID" -gt 0 ] && kill -0 "$DOCKER_PID" 2> /dev/null; then + starttime="$(date +%s)" + endtime="$(date +%s)" + + # Now wait for it to die + kill "$DOCKER_PID" + while kill -0 "$DOCKER_PID" 2> /dev/null; do + if [ $((endtime - starttime)) -le ${DOCKER_TIMEOUT} ]; then + sleep 1 + endtime=$(date +%s) + else + echo "[ERROR] Timeout while waiting for container docker to die" + exit 1 + fi + done + else + echo "[WARN] Your host might have been left with unreleased resources" + fi +} + +function run_build() { + local build_dir=$1 + local repository=$2 + local image=$3 + local version=$4 + local build_arch=$5 + local docker_cli=("${!6}") + + local push_images=() + + # Overwrites + if [ ! -z "$DOCKER_HUB" ]; then repository="$DOCKER_HUB"; fi + if [ ! -z "$IMAGE" ]; then image="$IMAGE"; fi + + # Init Cache + if [ "$DOCKER_CACHE" == "true" ]; then + echo "[INFO] Init cache for $repository/$image:$version" + if docker pull "$repository/$image:latest" > /dev/null 2>&1; then + docker_cli+=("--cache-from" "$repository/$image:latest") + else + docker_cli+=("--no-cache") + echo "[WARN] No cache image found. Cache is disabled for build" + fi + else + docker_cli+=("--no-cache") + fi + + # Build image + echo "[INFO] Run build for $repository/$image:$version" + docker build --pull -t "$repository/$image:$version" \ + --label "io.hass.version=$version" \ + --label "io.hass.arch=$build_arch" \ + -f "$TARGET/docker/Dockerfile.$build_arch" \ + "${docker_cli[@]}" \ + "$build_dir" + + echo "[INFO] Finish build for $repository/$image:$version" + docker tag "$repository/$image:$version" "$repository/$image:dev" +} + + +#### HassIO functions #### + +function build_addon() { + local build_arch=$1 + + local docker_cli=() + local image="" + local repository="" + local raw_image="" + local name="" + local description="" + local url="" + local args="" + + # Read addon config.json + name="$(jq --raw-output '.name // empty' "$TARGET/esphomeyaml/config.json" | sed "s/'//g")" + description="$(jq --raw-output '.description // empty' "$TARGET/esphomeyaml/config.json" | sed "s/'//g")" + url="$(jq --raw-output '.url // empty' "$TARGET/esphomeyaml/config.json")" + version="$(jq --raw-output '.version' "$TARGET/esphomeyaml/config.json")" + raw_image="$(jq --raw-output '.image // empty' "$TARGET/esphomeyaml/config.json")" + + # Read data from image + if [ ! -z "$raw_image" ]; then + repository="$(echo "$raw_image" | cut -f 1 -d '/')" + image="$(echo "$raw_image" | cut -f 2 -d '/')" + fi + + # Set additional labels + docker_cli+=("--label" "io.hass.name=$name") + docker_cli+=("--label" "io.hass.description=$description") + docker_cli+=("--label" "io.hass.type=addon") + + if [ ! -z "$url" ]; then + docker_cli+=("--label" "io.hass.url=$url") + fi + + # Start build + run_build "$TARGET" "$repository" "$image" "$version" \ + "$build_arch" docker_cli[@] +} + +#### initialized cross-build #### + +function init_crosscompile() { + echo "[INFO] Setup crosscompiling feature" + ( + mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc + update-binfmts --enable qemu-arm + update-binfmts --enable qemu-aarch64 + ) > /dev/null 2>&1 || echo "[WARN] Can't enable crosscompiling feature" +} + + +function clean_crosscompile() { + echo "[INFO] Clean crosscompiling feature" + if [ -f /proc/sys/fs/binfmt_misc ]; then + umount /proc/sys/fs/binfmt_misc || true + fi + + ( + update-binfmts --disable qemu-arm + update-binfmts --disable qemu-aarch64 + ) > /dev/null 2>&1 || echo "[WARN] No crosscompiling feature found for cleanup" +} + +#### Error handling #### + +function error_handling() { + stop_docker + clean_crosscompile + + exit 1 +} +trap 'error_handling' SIGINT SIGTERM + +#### Parse arguments #### + +while [[ $# -gt 0 ]]; do + key=$1 + case ${key} in + -h|--help) + print_help + ;; + -t|--target) + TARGET=$2 + shift + ;; + -i|--image) + IMAGE=$2 + shift + ;; + --no-cache) + DOCKER_CACHE="false" + ;; + -d|--docker-hub) + DOCKER_HUB=$2 + shift + ;; + --armhf) + BUILD_LIST+=("armhf") + ;; + --amd64) + BUILD_LIST+=("amd64") + ;; + --i386) + BUILD_LIST+=("i386") + ;; + --aarch64) + BUILD_LIST+=("aarch64") + ;; + --all) + BUILD_LIST=("armhf" "amd64" "i386" "aarch64") + ;; + + *) + echo "[WARN] $0 : Argument '$1' unknown will be Ignoring" + ;; + esac + shift +done + +# Check if an architecture is available +if [ "${#BUILD_LIST[@]}" -eq 0 ]; then + echo "[ERROR] You need select an architecture for build!" + exit 1 +fi + + +#### Main #### + +mkdir -p /data + +# Setup docker env +init_crosscompile +start_docker + +# Select arch build +for arch in "${BUILD_LIST[@]}"; do + (build_addon "$arch") & + BUILD_TASKS+=($!) +done + +# Wait until all build jobs are done +wait "${BUILD_TASKS[@]}" + +# Cleanup docker env +clean_crosscompile +stop_docker + +exit 0 diff --git a/docker/platformio-esp8266.ini b/docker/platformio-esp8266.ini new file mode 100644 index 0000000000..ed48f777cf --- /dev/null +++ b/docker/platformio-esp8266.ini @@ -0,0 +1,7 @@ +; This file allows the docker build file to install the required platformio +; platforms + +[env:espressif8266] +platform = espressif8266 +board = nodemcuv2 +framework = arduino diff --git a/docker/platformio.ini b/docker/platformio.ini index 75a9f25987..7f6ab6851d 100644 --- a/docker/platformio.ini +++ b/docker/platformio.ini @@ -1,12 +1,12 @@ ; This file allows the docker build file to install the required platformio ; platforms -[env:espressif32] -platform = espressif32 -board = nodemcu-32s -framework = arduino - [env:espressif8266] platform = espressif8266 board = nodemcuv2 framework = arduino + +[env:espressif32] +platform = espressif32 +board = nodemcu-32s +framework = arduino diff --git a/esphomeyaml-edge/build.json b/esphomeyaml-edge/build.json index a306cb0cc3..24268078ce 100644 --- a/esphomeyaml-edge/build.json +++ b/esphomeyaml-edge/build.json @@ -1,7 +1,7 @@ { "squash": false, "build_from": { - "aarch64": "arm64v8/ubuntu:bionic", + "aarch64": "homeassistant/aarch64-base:latest", "amd64": "ubuntu:bionic", "armhf": "homeassistant/armhf-base:latest", "i386": "i386/ubuntu:bionic" diff --git a/esphomeyaml/Dockerfile b/esphomeyaml/Dockerfile deleted file mode 100644 index 26759124c1..0000000000 --- a/esphomeyaml/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -# Dockerfile for HassIO add-on -ARG BUILD_FROM=ubuntu:bionic -FROM ${BUILD_FROM} - -# Re-declare BUILD_FROM to fix weird docker issue -ARG BUILD_FROM -ARG BUILD_VERSION - -# On amd64 and alike, using ubuntu as the base is better as building -# for the ESP32 only works with glibc (and ubuntu). However, on armhf -# the build toolchain frequently procudes segfaults under ubuntu. -# -> Use ubuntu for most architectures, except alpine for armhf -# -# * python and related required because this is a python project -# * git required for platformio library dependencies downloads -# * libc6-compat and openssh required on alpine for weird reasons -# * disable platformio telemetry on install -RUN /bin/bash -c "if [[ '$BUILD_FROM' = *\"ubuntu\"* ]]; then \ - apt-get update && apt-get install -y --no-install-recommends \ - python python-pip python-setuptools git && \ - rm -rf /var/lib/apt/lists/* /tmp/*; \ - else \ - apk add --no-cache python2 py2-pip git openssh libc6-compat; \ - fi" && \ - pip install --no-cache-dir platformio && \ - platformio settings set enable_telemetry No - - -# Create fake project to make platformio install all depdencies. -# * Ignore build errors from platformio - empty project -# * On alpine, only install ESP8266 toolchain -COPY platformio.ini /pio/platformio.ini -RUN /bin/bash -c "if [[ '$BUILD_FROM' = *\"ubuntu\"* ]]; then \ - platformio run -e espressif32 -e espressif8266 -d /pio; \ - else \ - echo \"\$(head -8 /pio/platformio.ini)\" >/pio/platformio.ini; \ - platformio run -e espressif8266 -d /pio; \ - fi"; exit 0 - -# Install latest esphomeyaml from git -RUN pip install --no-cache-dir \ - esphomeyaml==${BUILD_VERSION} - -CMD ["esphomeyaml", "/config/esphomeyaml", "dashboard"] diff --git a/esphomeyaml/build.json b/esphomeyaml/build.json deleted file mode 100644 index a306cb0cc3..0000000000 --- a/esphomeyaml/build.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "squash": false, - "build_from": { - "aarch64": "arm64v8/ubuntu:bionic", - "amd64": "ubuntu:bionic", - "armhf": "homeassistant/armhf-base:latest", - "i386": "i386/ubuntu:bionic" - }, - "args": {} -}