Automate HassIO builds using Gitlab CI (#52)

* Add Gitlab CI * Fix aarch64 edge build
2024-11-21 11:37:27 +01:00 · 2018-06-13 11:13:53 +02:00 · 2018-06-13 11:13:53 +02:00 · 9bb06782b2
commit 9bb06782b2
parent a827b51887
14 changed files with 577 additions and 60 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -106,3 +106,4 @@ venv.bak/
 config/
 examples/
 Dockerfile
+.git/
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -0,0 +1,131 @@
+---
+# Based on https://gitlab.com/hassio-addons/addon-node-red/blob/master/.gitlab-ci.yml
+variables:
+  DOCKER_DRIVER: overlay2
+
+stages:
+  - lint
+  - build
+  - deploy
+
+.lint: &lint
+  stage: lint
+  tags:
+    - python2.7
+    - esphomeyaml-lint
+
+.hassio-builder: &hassio-builder
+  before_script:
+    - docker info
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
+  services:
+    - docker:dind
+  tags:
+    - hassio-builder
+
+flake8:
+  <<: *lint
+  script:
+    - flake8 esphomeyaml
+
+pylint:
+  <<: *lint
+  script:
+    - pylint esphomeyaml
+
+.build: &build
+  <<: *hassio-builder
+  stage: build
+  script:
+    - |
+      hassio-builder.sh \
+           -t . \
+           -i ottowinter/esphomeyaml-hassio-${ADDON_ARCH} \
+           -d "$CI_REGISTRY" \
+           --${ADDON_ARCH}
+    - |
+      docker tag \
+        "${CI_REGISTRY}/ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:dev" \
+        "${CI_REGISTRY}/ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:${CI_COMMIT_SHA}"
+    - docker push "${CI_REGISTRY}/ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:${CI_COMMIT_SHA}"
+    - docker push "${CI_REGISTRY}/ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:dev"
+
+# Generic deploy template
+.deploy: &deploy
+  <<: *hassio-builder
+  stage: deploy
+  script:
+    - version=${CI_COMMIT_TAG:1}
+    - echo "Publishing version ${version}"
+    - docker login -u "$DOCKER_USER" -p "$DOCKER_PASSWORD"
+    - docker pull "${CI_REGISTRY}/ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:${CI_COMMIT_SHA}"
+    - |
+      docker tag \
+        "${CI_REGISTRY}/ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:${CI_COMMIT_SHA}" \
+        "ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:${version}"
+    - |
+      docker tag \
+        "ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:${version}" \
+        "ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:latest"
+    - docker push "ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:${version}"
+    - docker push "ottowinter/esphomeyaml-hassio-${ADDON_ARCH}:latest"
+
+# Build jobs
+build:armhf:
+  <<: *build
+  variables:
+    ADDON_ARCH: armhf
+
+build:aarch64:
+   <<: *build
+   variables:
+     ADDON_ARCH: aarch64
+
+build:i386:
+  <<: *build
+  variables:
+    ADDON_ARCH: i386
+
+build:amd64:
+  <<: *build
+  variables:
+    ADDON_ARCH: amd64
+
+# Deploy jobs
+deploy:armhf:
+  <<: *deploy
+  variables:
+    ADDON_ARCH: armhf
+  only:
+    - /^v\d+\.\d+\.\d+(?:(?:(?:\+|\.)?[a-zA-Z0-9]+)*)?$/
+  except:
+    - /^(?!master).+@/
+
+deploy:aarch64:
+  <<: *deploy
+  variables:
+    ADDON_ARCH: aarch64
+  only:
+    - /^v\d+\.\d+\.\d+(?:(?:(?:\+|\.)?[a-zA-Z0-9]+)*)?$/
+  except:
+    - /^(?!master).+@/
+
+deploy:i386:
+  <<: *deploy
+  variables:
+    ADDON_ARCH: i386
+  only:
+    - /^v\d+\.\d+\.\d+(?:(?:(?:\+|\.)?[a-zA-Z0-9]+)*)?$/
+  except:
+    - /^(?!master).+@/
+
+
+deploy:amd64:
+  <<: *deploy
+  variables:
+    ADDON_ARCH: amd64
+  only:
+    - /^v\d+\.\d+\.\d+(?:(?:(?:\+|\.)?[a-zA-Z0-9]+)*)?$/
+  except:
+    - /^(?!master).+@/
+
--- a/docker/Dockerfile.aarch64
+++ b/docker/Dockerfile.aarch64
@ -0,0 +1,20 @@
+# Dockerfile for aarch64 version of HassIO add-on
+FROM homeassistant/aarch64-base:latest
+
+RUN apk add --no-cache \
+        python2 \
+        py2-pip \
+        git \
+        openssh \
+        libc6-compat \
+    && \
+    pip install --no-cache-dir --no-binary :all: platformio && \
+    platformio settings set enable_telemetry No
+
+COPY docker/platformio-esp8266.ini /pio/platformio.ini
+RUN platformio run -d /pio; rm -rf /pio
+
+COPY . .
+RUN pip install --no-cache-dir --no-binary :all: -e .
+
+CMD ["esphomeyaml", "/config/esphomeyaml", "dashboard"]
--- a/docker/Dockerfile.amd64
+++ b/docker/Dockerfile.amd64
@ -0,0 +1,19 @@
+# Dockerfile for amd64 version of HassIO add-on
+FROM ubuntu:bionic
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        python \
+        python-pip \
+        python-setuptools \
+        git \
+    && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/*rm -rf /var/lib/apt/lists/* /tmp/* && \
+    pip install --no-cache-dir --no-binary :all: platformio && \
+    platformio settings set enable_telemetry No
+
+COPY docker/platformio.ini /pio/platformio.ini
+RUN platformio run -d /pio; rm -rf /pio
+
+COPY . .
+RUN pip install --no-cache-dir --no-binary :all: -e .
+
+CMD ["esphomeyaml", "/config/esphomeyaml", "dashboard"]
--- a/docker/Dockerfile.armhf
+++ b/docker/Dockerfile.armhf
@ -0,0 +1,20 @@
+# Dockerfile for armhf version of HassIO add-on
+FROM homeassistant/armhf-base:latest
+
+RUN apk add --no-cache \
+        python2 \
+        py2-pip \
+        git \
+        openssh \
+        libc6-compat \
+    && \
+    pip install --no-cache-dir --no-binary :all: platformio && \
+    platformio settings set enable_telemetry No
+
+COPY docker/platformio-esp8266.ini /pio/platformio.ini
+RUN platformio run -d /pio; rm -rf /pio
+
+COPY . .
+RUN pip install --no-cache-dir --no-binary :all: -e .
+
+CMD ["esphomeyaml", "/config/esphomeyaml", "dashboard"]
--- a/docker/Dockerfile.builder
+++ b/docker/Dockerfile.builder
@ -0,0 +1,32 @@
+FROM multiarch/ubuntu-core:amd64-xenial
+
+# setup locals
+RUN apt-get update && apt-get install -y \
+        jq \
+        git \
+        python3-setuptools \
+    && rm -rf /var/lib/apt/lists/* \
+ENV LANG C.UTF-8
+
+# Install docker
+# https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/
+RUN apt-get update && apt-get install -y \
+        apt-transport-https \
+        ca-certificates \
+        curl \
+        software-properties-common \
+    && rm -rf /var/lib/apt/lists/* \
+    && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
+    && add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \
+    && apt-get update && apt-get install -y docker-ce \
+    && rm -rf /var/lib/apt/lists/*
+
+# setup arm binary support
+RUN apt-get update && apt-get install -y \
+        qemu-user-static \
+        binfmt-support \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY hassio-builder.sh /usr/bin/
+
+WORKDIR /data
--- a/docker/Dockerfile.i386
+++ b/docker/Dockerfile.i386
@ -0,0 +1,19 @@
+# Dockerfile for i386 version of HassIO add-on
+FROM i386/ubuntu:bionic
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        python \
+        python-pip \
+        python-setuptools \
+        git \
+    && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/*rm -rf /var/lib/apt/lists/* /tmp/* && \
+    pip install --no-cache-dir --no-binary :all: platformio && \
+    platformio settings set enable_telemetry No
+
+COPY docker/platformio.ini /pio/platformio.ini
+RUN platformio run -d /pio; rm -rf /pio
+
+COPY . .
+RUN pip install --no-cache-dir --no-binary :all: -e .
+
+CMD ["esphomeyaml", "/config/esphomeyaml", "dashboard"]
--- a/docker/Dockerfile.lint
+++ b/docker/Dockerfile.lint
@ -0,0 +1,4 @@
+FROM python:2.7
+
+RUN pip install -r requirements.txt && \
+    pip install flake8==3.5.0 pylint==1.8.4
--- a/docker/hassio-builder.sh
+++ b/docker/hassio-builder.sh
@ -0,0 +1,318 @@
+#!/usr/bin/env bash
+# Based on Home Assistant's docker builder
+######################
+# Hass.io Build-env
+######################
+set -e
+
+echo -- "$@"
+
+#### Variable ####
+
+DOCKER_TIMEOUT=20
+DOCKER_PID=-1
+DOCKER_HUB=""
+DOCKER_CACHE="true"
+DOCKER_LOCAL="false"
+TARGET=""
+IMAGE=""
+BUILD_LIST=()
+BUILD_TASKS=()
+
+#### Misc functions ####
+
+function print_help() {
+    cat << EOF
+Hass.io build-env for ecosystem:
+docker run --rm homeassistant/{arch}-builder:latest [options]
+
+Options:
+  -h, --help
+        Display this help and exit.
+
+  Repository / Data
+    -t, --target <PATH_TO_BUILD>
+        Set local folder or path inside repository for build.
+
+  Version/Image handling
+    -i, --image <IMAGE_NAME>
+        Overwrite image name of build / support {arch}
+
+  Architecture
+    --armhf
+        Build for arm.
+    --amd64
+        Build for intel/amd 64bit.
+    --aarch64
+        Build for arm 64bit.
+    --i386
+        Build for intel/amd 32bit.
+    --all
+        Build all architecture.
+
+  Build handling
+    --no-cache
+       Disable cache for the build (from latest).
+    -d, --docker-hub <DOCKER_REPOSITORY>
+       Set or overwrite the docker repository.
+
+    Use the host docker socket if mapped into container:
+       /var/run/docker.sock
+
+EOF
+
+    exit 1
+}
+
+#### Docker functions ####
+
+function start_docker() {
+    local starttime
+    local endtime
+
+    if [ -S "/var/run/docker.sock" ]; then
+        echo "[INFO] Use host docker setup with '/var/run/docker.sock'"
+        DOCKER_LOCAL="true"
+        return 0
+    fi
+
+    echo "[INFO] Starting docker."
+    dockerd 2> /dev/null &
+    DOCKER_PID=$!
+
+    echo "[INFO] Waiting for docker to initialize..."
+    starttime="$(date +%s)"
+    endtime="$(date +%s)"
+    until docker info >/dev/null 2>&1; do
+        if [ $((endtime - starttime)) -le ${DOCKER_TIMEOUT} ]; then
+            sleep 1
+            endtime=$(date +%s)
+        else
+            echo "[ERROR] Timeout while waiting for docker to come up"
+            exit 1
+        fi
+    done
+    echo "[INFO] Docker was initialized"
+}
+
+
+function stop_docker() {
+    local starttime
+    local endtime
+
+    if [ "$DOCKER_LOCAL" == "true" ]; then
+        return 0
+    fi
+
+    echo "[INFO] Stopping in container docker..."
+    if [ "$DOCKER_PID" -gt 0 ] && kill -0 "$DOCKER_PID" 2> /dev/null; then
+        starttime="$(date +%s)"
+        endtime="$(date +%s)"
+
+        # Now wait for it to die
+        kill "$DOCKER_PID"
+        while kill -0 "$DOCKER_PID" 2> /dev/null; do
+            if [ $((endtime - starttime)) -le ${DOCKER_TIMEOUT} ]; then
+                sleep 1
+                endtime=$(date +%s)
+            else
+                echo "[ERROR] Timeout while waiting for container docker to die"
+                exit 1
+            fi
+        done
+    else
+        echo "[WARN] Your host might have been left with unreleased resources"
+    fi
+}
+
+function run_build() {
+    local build_dir=$1
+    local repository=$2
+    local image=$3
+    local version=$4
+    local build_arch=$5
+    local docker_cli=("${!6}")
+
+    local push_images=()
+
+    # Overwrites
+    if [ ! -z "$DOCKER_HUB" ]; then repository="$DOCKER_HUB"; fi
+    if [ ! -z "$IMAGE" ]; then image="$IMAGE"; fi
+
+    # Init Cache
+    if [ "$DOCKER_CACHE" == "true" ]; then
+        echo "[INFO] Init cache for $repository/$image:$version"
+        if docker pull "$repository/$image:latest" > /dev/null 2>&1; then
+            docker_cli+=("--cache-from" "$repository/$image:latest")
+        else
+            docker_cli+=("--no-cache")
+            echo "[WARN] No cache image found. Cache is disabled for build"
+        fi
+    else
+        docker_cli+=("--no-cache")
+    fi
+
+    # Build image
+    echo "[INFO] Run build for $repository/$image:$version"
+    docker build --pull -t "$repository/$image:$version" \
+        --label "io.hass.version=$version" \
+        --label "io.hass.arch=$build_arch" \
+        -f "$TARGET/docker/Dockerfile.$build_arch" \
+        "${docker_cli[@]}" \
+        "$build_dir"
+
+    echo "[INFO] Finish build for $repository/$image:$version"
+    docker tag "$repository/$image:$version" "$repository/$image:dev"
+}
+
+
+#### HassIO functions ####
+
+function build_addon() {
+    local build_arch=$1
+
+    local docker_cli=()
+    local image=""
+    local repository=""
+    local raw_image=""
+    local name=""
+    local description=""
+    local url=""
+    local args=""
+
+    # Read addon config.json
+    name="$(jq --raw-output '.name // empty' "$TARGET/esphomeyaml/config.json" | sed "s/'//g")"
+    description="$(jq --raw-output '.description // empty' "$TARGET/esphomeyaml/config.json" | sed "s/'//g")"
+    url="$(jq --raw-output '.url // empty' "$TARGET/esphomeyaml/config.json")"
+    version="$(jq --raw-output '.version' "$TARGET/esphomeyaml/config.json")"
+    raw_image="$(jq --raw-output '.image // empty' "$TARGET/esphomeyaml/config.json")"
+
+    # Read data from image
+    if [ ! -z "$raw_image" ]; then
+        repository="$(echo "$raw_image" | cut -f 1 -d '/')"
+        image="$(echo "$raw_image" | cut -f 2 -d '/')"
+    fi
+
+    # Set additional labels
+    docker_cli+=("--label" "io.hass.name=$name")
+    docker_cli+=("--label" "io.hass.description=$description")
+    docker_cli+=("--label" "io.hass.type=addon")
+
+    if [ ! -z "$url" ]; then
+        docker_cli+=("--label" "io.hass.url=$url")
+    fi
+
+    # Start build
+    run_build "$TARGET" "$repository" "$image" "$version" \
+        "$build_arch" docker_cli[@]
+}
+
+#### initialized cross-build ####
+
+function init_crosscompile() {
+    echo "[INFO] Setup crosscompiling feature"
+    (
+        mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc
+        update-binfmts --enable qemu-arm
+        update-binfmts --enable qemu-aarch64
+    ) > /dev/null 2>&1 || echo "[WARN] Can't enable crosscompiling feature"
+}
+
+
+function clean_crosscompile() {
+    echo "[INFO] Clean crosscompiling feature"
+    if [ -f /proc/sys/fs/binfmt_misc ]; then
+        umount /proc/sys/fs/binfmt_misc || true
+    fi
+
+    (
+        update-binfmts --disable qemu-arm
+        update-binfmts --disable qemu-aarch64
+    ) > /dev/null 2>&1 || echo "[WARN] No crosscompiling feature found for cleanup"
+}
+
+#### Error handling ####
+
+function error_handling() {
+    stop_docker
+    clean_crosscompile
+
+    exit 1
+}
+trap 'error_handling' SIGINT SIGTERM
+
+#### Parse arguments ####
+
+while [[ $# -gt 0 ]]; do
+    key=$1
+    case ${key} in
+        -h|--help)
+            print_help
+            ;;
+        -t|--target)
+            TARGET=$2
+            shift
+            ;;
+        -i|--image)
+            IMAGE=$2
+            shift
+            ;;
+        --no-cache)
+            DOCKER_CACHE="false"
+            ;;
+        -d|--docker-hub)
+            DOCKER_HUB=$2
+            shift
+            ;;
+        --armhf)
+            BUILD_LIST+=("armhf")
+            ;;
+        --amd64)
+            BUILD_LIST+=("amd64")
+            ;;
+        --i386)
+            BUILD_LIST+=("i386")
+            ;;
+        --aarch64)
+            BUILD_LIST+=("aarch64")
+            ;;
+        --all)
+            BUILD_LIST=("armhf" "amd64" "i386" "aarch64")
+            ;;
+
+        *)
+            echo "[WARN] $0 : Argument '$1' unknown will be Ignoring"
+            ;;
+    esac
+    shift
+done
+
+# Check if an architecture is available
+if [ "${#BUILD_LIST[@]}" -eq 0 ]; then
+    echo "[ERROR] You need select an architecture for build!"
+    exit 1
+fi
+
+
+#### Main ####
+
+mkdir -p /data
+
+# Setup docker env
+init_crosscompile
+start_docker
+
+# Select arch build
+for arch in "${BUILD_LIST[@]}"; do
+    (build_addon "$arch") &
+    BUILD_TASKS+=($!)
+done
+
+# Wait until all build jobs are done
+wait "${BUILD_TASKS[@]}"
+
+# Cleanup docker env
+clean_crosscompile
+stop_docker
+
+exit 0
--- a/docker/platformio-esp8266.ini
+++ b/docker/platformio-esp8266.ini
@ -0,0 +1,7 @@
+; This file allows the docker build file to install the required platformio
+; platforms
+
+[env:espressif8266]
+platform = espressif8266
+board = nodemcuv2
+framework = arduino
--- a/docker/platformio.ini
+++ b/docker/platformio.ini
@ -1,12 +1,12 @@
 ; This file allows the docker build file to install the required platformio
 ; platforms

-[env:espressif32]
-platform = espressif32
-board = nodemcu-32s
-framework = arduino
-
 [env:espressif8266]
 platform = espressif8266
 board = nodemcuv2
 framework = arduino
+
+[env:espressif32]
+platform = espressif32
+board = nodemcu-32s
+framework = arduino
--- a/esphomeyaml-edge/build.json
+++ b/esphomeyaml-edge/build.json
@ -1,7 +1,7 @@
 {
    "squash": false,
    "build_from": {
-        "aarch64": "arm64v8/ubuntu:bionic",
+        "aarch64": "homeassistant/aarch64-base:latest",
        "amd64": "ubuntu:bionic",
        "armhf": "homeassistant/armhf-base:latest",
        "i386": "i386/ubuntu:bionic"
--- a/esphomeyaml/Dockerfile
+++ b/esphomeyaml/Dockerfile
@ -1,44 +0,0 @@
-# Dockerfile for HassIO add-on
-ARG BUILD_FROM=ubuntu:bionic
-FROM ${BUILD_FROM}
-
-# Re-declare BUILD_FROM to fix weird docker issue
-ARG BUILD_FROM
-ARG BUILD_VERSION
-
-# On amd64 and alike, using ubuntu as the base is better as building
-# for the ESP32 only works with glibc (and ubuntu). However, on armhf
-# the build toolchain frequently procudes segfaults under ubuntu.
-#  -> Use ubuntu for most architectures, except alpine for armhf
-#
-# * python and related required because this is a python project
-# * git required for platformio library dependencies downloads
-# * libc6-compat and openssh required on alpine for weird reasons
-# * disable platformio telemetry on install
-RUN /bin/bash -c "if [[ '$BUILD_FROM' = *\"ubuntu\"* ]]; then \
-      apt-get update && apt-get install -y --no-install-recommends \
-        python python-pip python-setuptools git && \
-      rm -rf /var/lib/apt/lists/* /tmp/*; \
-    else \
-       apk add --no-cache python2 py2-pip git openssh libc6-compat; \
-    fi" && \
-    pip install --no-cache-dir platformio && \
-    platformio settings set enable_telemetry No
-
-
-# Create fake project to make platformio install all depdencies.
-# * Ignore build errors from platformio - empty project
-# * On alpine, only install ESP8266 toolchain
-COPY platformio.ini /pio/platformio.ini
-RUN /bin/bash -c "if [[ '$BUILD_FROM' = *\"ubuntu\"* ]]; then \
-      platformio run -e espressif32 -e espressif8266 -d /pio; \
-    else \
-      echo \"\$(head -8 /pio/platformio.ini)\" >/pio/platformio.ini; \
-      platformio run -e espressif8266 -d /pio; \
-    fi"; exit 0
-
-# Install latest esphomeyaml from git
-RUN pip install --no-cache-dir \
-      esphomeyaml==${BUILD_VERSION}
-
-CMD ["esphomeyaml", "/config/esphomeyaml", "dashboard"]
--- a/esphomeyaml/build.json
+++ b/esphomeyaml/build.json
@ -1,10 +0,0 @@
-{
-    "squash": false,
-    "build_from": {
-        "aarch64": "arm64v8/ubuntu:bionic",
-        "amd64": "ubuntu:bionic",
-        "armhf": "homeassistant/armhf-base:latest",
-        "i386": "i386/ubuntu:bionic"
-    },
-    "args": {}
-}