Merge pull request #13421 from ninjadq/remove_clair

Remove clair

Makefile

@@ -76,7 +76,6 @@ REGISTRYSERVER=
 REGISTRYPROJECTNAME=goharbor
 DEVFLAG=true
 NOTARYFLAG=false
-CLAIRFLAG=false
 TRIVYFLAG=false
 HTTPPROXY=
 BUILDBIN=false
@@ -101,9 +100,7 @@ PREPARE_VERSION_NAME=versions
 #versions
 REGISTRYVERSION=v2.7.1-patch-2819-2553-redis
 NOTARYVERSION=v0.6.1
-CLAIRVERSION=v2.1.6
 NOTARYMIGRATEVERSION=v3.5.4
-CLAIRADAPTERVERSION=v1.1.1
 TRIVYVERSION=v0.9.2
 TRIVYADAPTERVERSION=v0.14.1
 
@@ -117,11 +114,9 @@ CHARTMUSEUM_SRC_TAG=v0.12.0
 REGISTRY_SRC_TAG=v2.7.1
 
 # dependency binaries
-CLAIRURL=https://storage.googleapis.com/harbor-builds/bin/clair/release2.0-${CLAIRVERSION}/clair
 CHARTURL=https://storage.googleapis.com/harbor-builds/bin/chartmuseum/release-${CHARTMUSEUMVERSION}/chartm
 NORARYURL=https://storage.googleapis.com/harbor-builds/bin/notary/release-${NOTARYVERSION}/binary-bundle.tgz
 REGISTRYURL=https://storage.googleapis.com/harbor-builds/bin/registry/release-${REGISTRYVERSION}/registry
-CLAIR_ADAPTER_DOWNLOAD_URL=https://github.com/goharbor/harbor-scanner-clair/releases/download/$(CLAIRADAPTERVERSION)/harbor-scanner-clair_$(CLAIRADAPTERVERSION:v%=%)_Linux_x86_64.tar.gz
 TRIVY_DOWNLOAD_URL=https://github.com/aquasecurity/trivy/releases/download/$(TRIVYVERSION)/trivy_$(TRIVYVERSION:v%=%)_Linux-64bit.tar.gz
 TRIVY_ADAPTER_DOWNLOAD_URL=https://github.com/aquasecurity/harbor-scanner-trivy/releases/download/$(TRIVYADAPTERVERSION)/harbor-scanner-trivy_$(TRIVYADAPTERVERSION:v%=%)_Linux_x86_64.tar.gz
 
@@ -129,8 +124,6 @@ define VERSIONS_FOR_PREPARE
 VERSION_TAG: $(VERSIONTAG)
 REGISTRY_VERSION: $(REGISTRYVERSION)
 NOTARY_VERSION: $(NOTARYVERSION)
-CLAIR_VERSION: $(CLAIRVERSION)
-CLAIR_ADAPTER_VERSION: $(CLAIRADAPTERVERSION)
 TRIVY_VERSION: $(TRIVYVERSION)
 TRIVY_ADAPTER_VERSION: $(TRIVYADAPTERVERSION)
 CHARTMUSEUM_VERSION: $(CHARTMUSEUMVERSION)
@@ -210,9 +203,6 @@ PREPARECMD_PARA=--conf $(INSIDE_CONFIGPATH)/$(CONFIGFILE)
 ifeq ($(NOTARYFLAG), true)
 	PREPARECMD_PARA+= --with-notary
 endif
-ifeq ($(CLAIRFLAG), true)
-	PREPARECMD_PARA+= --with-clair
-endif
 ifeq ($(TRIVYFLAG), true)
 	PREPARECMD_PARA+= --with-trivy
 endif
@@ -239,14 +229,7 @@ DOCKERIMAGENAME_REGCTL=goharbor/harbor-registryctl
 
 # docker-compose files
 DOCKERCOMPOSEFILEPATH=$(MAKEPATH)
-DOCKERCOMPOSETPLFILENAME=docker-compose.tpl
 DOCKERCOMPOSEFILENAME=docker-compose.yml
-DOCKERCOMPOSENOTARYTPLFILENAME=docker-compose.notary.tpl
-DOCKERCOMPOSENOTARYFILENAME=docker-compose.notary.yml
-DOCKERCOMPOSECLAIRTPLFILENAME=docker-compose.clair.tpl
-DOCKERCOMPOSECLAIRFILENAME=docker-compose.clair.yml
-DOCKERCOMPOSECHARTMUSEUMTPLFILENAME=docker-compose.chartmuseum.tpl
-DOCKERCOMPOSECHARTMUSEUMFILENAME=docker-compose.chartmuseum.yml
 
 SEDCMD=$(shell which sed)
 SEDCMDI=$(SEDCMD) -i
@@ -297,9 +280,6 @@ DOCKERCOMPOSE_FILE_OPT=-f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
 ifeq ($(NOTARYFLAG), true)
 	DOCKERSAVE_PARA+= goharbor/notary-server-photon:$(VERSIONTAG) goharbor/notary-signer-photon:$(VERSIONTAG)
 endif
-ifeq ($(CLAIRFLAG), true)
-	DOCKERSAVE_PARA+= goharbor/clair-photon:$(VERSIONTAG) goharbor/clair-adapter-photon:$(VERSIONTAG)
-endif
 ifeq ($(TRIVYFLAG), true)
 	DOCKERSAVE_PARA+= goharbor/trivy-adapter-photon:$(VERSIONTAG)
 endif
@@ -409,18 +389,18 @@ build:
 	 -e REGISTRYVERSION=$(REGISTRYVERSION) -e REGISTRY_SRC_TAG=$(REGISTRY_SRC_TAG) \
 	 -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
 	 -e TRIVYVERSION=$(TRIVYVERSION) -e TRIVYADAPTERVERSION=$(TRIVYADAPTERVERSION) \
-	 -e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRADAPTERVERSION=$(CLAIRADAPTERVERSION) -e VERSIONTAG=$(VERSIONTAG) \
+	 -e VERSIONTAG=$(VERSIONTAG) \
 	 -e BUILDBIN=$(BUILDBIN) \
 	 -e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e CHARTMUSEUM_SRC_TAG=$(CHARTMUSEUM_SRC_TAG) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER) \
 	 -e NPM_REGISTRY=$(NPM_REGISTRY) -e BASEIMAGETAG=$(BASEIMAGETAG) -e BASEIMAGENAMESPACE=$(BASEIMAGENAMESPACE) \
-	 -e CLAIRURL=$(CLAIRURL) -e CHARTURL=$(CHARTURL) -e NORARYURL=$(NORARYURL) -e REGISTRYURL=$(REGISTRYURL) -e CLAIR_ADAPTER_DOWNLOAD_URL=$(CLAIR_ADAPTER_DOWNLOAD_URL) \
+	 -e CHARTURL=$(CHARTURL) -e NORARYURL=$(NORARYURL) -e REGISTRYURL=$(REGISTRYURL) \
 	 -e TRIVY_DOWNLOAD_URL=$(TRIVY_DOWNLOAD_URL) -e TRIVY_ADAPTER_DOWNLOAD_URL=$(TRIVY_ADAPTER_DOWNLOAD_URL)
 
 build_standalone_db_migrator: compile_standalone_db_migrator
 	make -f $(MAKEFILEPATH_PHOTON)/Makefile _build_standalone_db_migrator -e BASEIMAGETAG=$(BASEIMAGETAG) -e VERSIONTAG=$(VERSIONTAG)
 
 build_base_docker:
-	@for name in chartserver clair clair-adapter trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
+	@for name in chartserver trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
 		echo $$name ; \
 		$(DOCKERBUILD) --pull --no-cache -f $(MAKEFILEPATH_PHOTON)/$$name/Dockerfile.base -t $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) --label base-build-date=$(date +"%Y%m%d") . && \
 		if [ -n "$(PUSHBASEIMAGE)" ] ; then \
@@ -429,7 +409,7 @@ build_base_docker:
 	done
 
 pull_base_docker:
-	@for name in chartserver clair clair-adapter trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
+	@for name in chartserver trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
 		echo $$name ; \
 		$(DOCKERPULL) $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) ; \
 	done

make/harbor.yml.tmpl

@@ -61,11 +61,6 @@ data_volume: /data
 #   redirect:
 #     disabled: false
 
-# Clair configuration
-clair:
-  # The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
-  updaters_interval: 12
-
 # Trivy configuration
 #
 # Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases.
@@ -147,13 +142,6 @@ _version: 2.0.0
 #     ssl_mode: disable
 #     max_idle_conns: 2
 #     max_open_conns: 0
-#   clair:
-#     host: clair_db_host
-#     port: clair_db_port
-#     db_name: clair_db_name
-#     username: clair_db_username
-#     password: clair_db_password
-#     ssl_mode: disable
 #   notary_signer:
 #     host: notary_signer_db_host
 #     port: notary_signer_db_port
@@ -206,7 +194,6 @@ proxy:
   components:
     - core
     - jobservice
-    - clair
     - trivy
 
 # metric:

make/install.sh

@@ -9,14 +9,13 @@ set +o noglob
 
 usage=$'Please set hostname and other necessary attributes in harbor.yml first. DO NOT use localhost or 127.0.0.1 for hostname, because Harbor needs to be accessed by external clients.
 Please set --with-notary if needs enable Notary in Harbor, and set ui_url_protocol/ssl_cert/ssl_cert_key in harbor.yml bacause notary must run under https. 
-Please set --with-clair if needs enable Clair in Harbor
 Please set --with-trivy if needs enable Trivy in Harbor
 Please set --with-chartmuseum if needs enable Chartmuseum in Harbor'
 item=0
 
 # notary is not enabled by default
 with_notary=$false
-# clair is not enabled by default
+# clair is deprecated
 with_clair=$false
 # trivy is not enabled by default
 with_trivy=$false
@@ -43,6 +42,12 @@ while [ $# -gt 0 ]; do
         shift || true
 done
 
+if [ $with_clair ]
+then
+    error "Clair is deprecated please remove it from installation arguments !!!"
+    exit 1
+fi
+
 workdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd $workdir
 
@@ -71,10 +76,6 @@ if [ $with_notary ]
 then
     prepare_para="${prepare_para} --with-notary"
 fi
-if [ $with_clair ]
-then
-    prepare_para="${prepare_para} --with-clair"
-fi
 if [ $with_trivy ]
 then
     prepare_para="${prepare_para} --with-trivy"

make/photon/Makefile

@@ -59,14 +59,6 @@ DOCKERFILEPATH_POSTGRESQL=$(DOCKERFILEPATH)/postgresql
 DOCKERFILENAME_POSTGRESQL=Dockerfile
 DOCKERIMAGENAME_POSTGRESQL=goharbor/postgresql-photon
 
-DOCKERFILEPATH_CLAIR=$(DOCKERFILEPATH)/clair
-DOCKERFILENAME_CLAIR=Dockerfile
-DOCKERIMAGENAME_CLAIR=goharbor/clair-photon
-
-DOCKERFILEPATH_CLAIR_ADAPTER=$(DOCKERFILEPATH)/clair-adapter
-DOCKERFILENAME_CLAIR_ADAPTER=Dockerfile
-DOCKERIMAGENAME_CLAIR_ADAPTER=goharbor/clair-adapter-photon
-
 DOCKERFILEPATH_TRIVY_ADAPTER=$(DOCKERFILEPATH)/trivy-adapter
 DOCKERFILENAME_TRIVY_ADAPTER=Dockerfile
 DOCKERIMAGENAME_TRIVY_ADAPTER=goharbor/trivy-adapter-photon
@@ -137,38 +129,6 @@ _build_log:
 	$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_LOG)/$(DOCKERFILENAME_LOG) -t $(DOCKERIMAGENAME_LOG):$(VERSIONTAG) .
 	@echo "Done."
 
-_build_clair:
-	@if [ "$(CLAIRFLAG)" = "true" ] ; then \
-		if [ "$(BUILDBIN)" != "true" ] ; then \
-			rm -rf $(DOCKERFILEPATH_CLAIR)/binary && mkdir -p $(DOCKERFILEPATH_CLAIR)/binary && \
-			$(call _get_binary, $(CLAIRURL), $(DOCKERFILEPATH_CLAIR)/binary/clair); \
-		else \
-			cd $(DOCKERFILEPATH_CLAIR) && $(DOCKERFILEPATH_CLAIR)/builder $(CLAIRVERSION) && cd - ; \
-		fi ; \
-		echo "building clair container for photon..." ; \
-		$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_CLAIR)/$(DOCKERFILENAME_CLAIR) -t $(DOCKERIMAGENAME_CLAIR):$(VERSIONTAG) . ; \
-		rm -rf $(DOCKERFILEPATH_CLAIR)/binary; \
-		echo "Done." ; \
-	fi
-
-_build_clair_adapter:
-	@if [ "$(CLAIRFLAG)" = "true" ] ; then \
-		if [ "$(BUILDBIN)" != "true" ] ; then \
-			rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary && mkdir -p $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary && \
-			$(call _extract_archive, $(CLAIR_ADAPTER_DOWNLOAD_URL), $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/) && \
-			mv $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/scanner-clair $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/harbor-scanner-clair; \
-		else \
-			cd $(DOCKERFILEPATH_CLAIR_ADAPTER) && $(DOCKERFILEPATH_CLAIR_ADAPTER)/builder.sh $(CLAIRADAPTERVERSION) && cd - ; \
-		fi ; \
-		echo "Building Clair adapter container for photon..." ; \
-		$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) \
-			--build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) \
-			-f $(DOCKERFILEPATH_CLAIR_ADAPTER)/$(DOCKERFILENAME_CLAIR_ADAPTER) \
-			-t $(DOCKERIMAGENAME_CLAIR_ADAPTER):$(VERSIONTAG) . ; \
-		rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary; \
-		echo "Done." ; \
-	fi
-
 _build_trivy_adapter:
 	@if [ "$(TRIVYFLAG)" = "true" ] ; then \
 		rm -rf $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary && mkdir -p $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary ; \
@@ -263,7 +223,7 @@ define _get_binary
 	$(CURL) --connect-timeout 30 -f -k -L $1 -o $2 || exit 1
 endef
 
-build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_clair_adapter _build_trivy_adapter _build_redis _build_chart_server
+build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_trivy_adapter _build_redis _build_chart_server
 
 cleanimage:
 	@echo "cleaning image for photon..."

make/photon/clair-adapter/Dockerfile

@@ -1,18 +0,0 @@
-ARG harbor_base_image_version
-ARG harbor_base_namespace
-FROM ${harbor_base_namespace}/harbor-clair-adapter-base:${harbor_base_image_version}
-
-COPY ./make/photon/common/install_cert.sh /home/clair-adapter
-COPY ./make/photon/clair-adapter/entrypoint.sh /home/clair-adapter
-COPY ./make/photon/clair-adapter/binary/harbor-scanner-clair /clair-adapter/clair-adapter
-
-RUN chown -R clair-adapter:clair-adapter /etc/pki/tls/certs \
-    && chown -R clair-adapter:clair-adapter /clair-adapter && chmod u+x /clair-adapter/clair-adapter \
-    && chown clair-adapter:clair-adapter /home/clair-adapter/entrypoint.sh && chmod u+x /home/clair-adapter/entrypoint.sh \
-    && chown clair-adapter:clair-adapter /home/clair-adapter/install_cert.sh && chmod u+x /home/clair-adapter/install_cert.sh
-
-HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS http://127.0.0.1:8080/probe/healthy || curl -k -sS https://127.0.0.1:8443/probe/healthy || exit 1
-
-USER clair-adapter
-
-ENTRYPOINT ["/home/clair-adapter/entrypoint.sh"]

make/photon/clair-adapter/Dockerfile.base

@@ -1,7 +0,0 @@
-FROM photon:2.0
-
-RUN tdnf install -y shadow >> /dev/null \
-    && tdnf clean all \
-    && mkdir /clair-adapter/ \
-    && groupadd -r -g 10000 clair-adapter \
-    && useradd --no-log-init -m -r -g 10000 -u 10000 clair-adapter

make/photon/clair-adapter/Dockerfile.binary

@@ -1,11 +0,0 @@
-FROM golang:1.14.7
-
-ARG VERSION
-ARG COMMIT
-
-ADD .   /go/src/github.com/goharbor/harbor-scanner-clair/
-WORKDIR /go/src/github.com/goharbor/harbor-scanner-clair/
-
-RUN export GOFLAGS=-mod=vendor GOOS=linux GO111MODULE=on CGO_ENABLED=0 && \
-	go build -ldflags "-X main.version=$VERSION -X main.commit=$COMMIT -X main.date=`date -u --iso-8601=seconds`" \
-	-o harbor-scanner-clair cmd/harbor-scanner-clair/main.go

make/photon/clair-adapter/builder.sh

@@ -1,39 +0,0 @@
-#!/bin/bash
-
-set +e
-
-if [ -z $1 ]; then
-  error "Please set the 'version' variable"
-  exit 1
-fi
-
-VERSION="$1"
-
-set -e
-
-# the temp folder to store binary file...
-mkdir -p binary
-rm -rf binary/harbor-scanner-clair || true
-
-cd $(dirname $0)
-cur=$PWD
-
-# The temporary directory to clone Clair adapter source code
-TEMP=$(mktemp -d ${TMPDIR-/tmp}/clair-adapter.XXXXXX)
-git clone https://github.com/goharbor/harbor-scanner-clair.git $TEMP
-cd $TEMP; git checkout $VERSION; export COMMIT=$(git rev-list -1 HEAD); cd -
-
-echo "Building Clair adapter binary based on golang:1.14.7..."
-cp Dockerfile.binary $TEMP
-docker build --build-arg VERSION=${VERSION} --build-arg COMMIT=${COMMIT} -f $TEMP/Dockerfile.binary -t clair-adapter-golang $TEMP
-
-echo "Copying Clair adapter binary from the container to the local directory..."
-ID=$(docker create clair-adapter-golang)
-docker cp $ID:/go/src/github.com/goharbor/harbor-scanner-clair/harbor-scanner-clair binary
-
-docker rm -f $ID
-docker rmi -f clair-adapter-golang
-
-echo "Building Clair adapter binary finished successfully"
-cd $cur
-rm -rf $TEMP

make/photon/clair-adapter/entrypoint.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-
-set -e
-
-/home/clair-adapter/install_cert.sh
-
-/clair-adapter/clair-adapter

make/photon/clair/Dockerfile

@@ -1,22 +0,0 @@
-ARG harbor_base_image_version
-ARG harbor_base_namespace
-FROM ${harbor_base_namespace}/harbor-clair-base:${harbor_base_image_version}
-
-COPY ./make/photon/clair/binary/clair /home/clair/
-COPY ./make/photon/clair/docker-entrypoint.sh /home/clair/
-COPY ./make/photon/clair/dumb-init /home/clair/
-COPY ./make/photon/common/install_cert.sh /home/clair/
-
-VOLUME /config
-
-RUN chown -R clair:clair /etc/pki/tls/certs && chown -R clair:clair /home/clair \
-    && chmod u+x /home/clair/clair \
-    && chmod u+x /home/clair/docker-entrypoint.sh \
-    && chmod u+x /home/clair/install_cert.sh \
-    && chmod +x /home/clair/dumb-init
-
-HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS 127.0.0.1:6061/health || exit 1
-
-WORKDIR /home/clair
-USER clair
-ENTRYPOINT ["./docker-entrypoint.sh"]

make/photon/clair/Dockerfile.base

@@ -1,6 +0,0 @@
-FROM photon:2.0
-
-RUN tdnf install -y git shadow rpm xz python-xml >>/dev/null\
-    && tdnf clean all \
-    && groupadd -r -g 10000 clair \
-    && useradd --no-log-init -m -g 10000 -u 10000 clair

make/photon/clair/Dockerfile.binary

@@ -1,7 +0,0 @@
-FROM golang:1.14.7
-
-ADD .   /go/src/github.com/quay/clair/
-WORKDIR /go/src/github.com/quay/clair/
-
-RUN export CLAIR_VERSION=$(git describe --tag --always --dirty) GO111MODULE=on && \
-	go build -ldflags "-X github.com/quay/clair/pkg/version.Version=$CLAIR_VERSION" ./cmd/clair

make/photon/clair/builder

@@ -1,38 +0,0 @@
-#!/bin/bash
-
-set +e
-
-if [ -z $1 ]; then
-  error "Please set the 'version' variable"
-  exit 1
-fi
-
-VERSION="$1"
-
-set -e
-
-# the temp folder to store binary file...
-mkdir -p binary
-rm -rf binary/clair || true
-
-cd `dirname $0`
-cur=$PWD
-
-# the temp folder to store distribution source code...
-TEMP=`mktemp -d /$TMPDIR/clair.XXXXXX`
-git clone -b $VERSION --single-branch https://github.com/quay/clair.git $TEMP
-
-echo 'build the clair binary bases on the golang:1.14.7'
-cp Dockerfile.binary $TEMP
-docker build -f $TEMP/Dockerfile.binary -t clair-golang $TEMP
-
-echo 'copy the clair binary to local...'
-ID=$(docker create clair-golang)
-docker cp $ID:/go/src/github.com/quay/clair/clair binary
-
-docker rm -f $ID
-docker rmi -f clair-golang
-
-echo "Build clair binary success, then to build photon image..."
-cd $cur
-rm -rf $TEMP

make/photon/clair/docker-entrypoint.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-set -e
-
-/home/clair/install_cert.sh
-/home/clair/dumb-init -- /home/clair/clair -config /etc/clair/config.yaml $*
-
-set +e

make/photon/prepare/commands/prepare.py

@@ -15,8 +15,6 @@ from utils.registry_ctl import prepare_registry_ctl
 from utils.core import prepare_core
 from utils.notary import prepare_notary
 from utils.log import prepare_log_configs
-from utils.clair import prepare_clair
-from utils.clair_adapter import prepare_clair_adapter
 from utils.chart import prepare_chartmuseum
 from utils.docker_compose import prepare_docker_compose
 from utils.nginx import prepare_nginx, nginx_confd_dir
@@ -30,13 +28,12 @@ old_private_key_pem_path, old_crt_path)
 @click.command()
 @click.option('--conf', default=input_config_path, help="the path of Harbor configuration file")
 @click.option('--with-notary', is_flag=True, help="the Harbor instance is to be deployed with notary")
-@click.option('--with-clair', is_flag=True, help="the Harbor instance is to be deployed with clair")
 @click.option('--with-trivy', is_flag=True, help="the Harbor instance is to be deployed with Trivy")
 @click.option('--with-chartmuseum', is_flag=True, help="the Harbor instance is to be deployed with chart repository supporting")
-def prepare(conf, with_notary, with_clair, with_trivy, with_chartmuseum):
+def prepare(conf, with_notary, with_trivy, with_chartmuseum):
 
     delfile(config_dir)
-    config_dict = parse_yaml_config(conf, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
+    config_dict = parse_yaml_config(conf, with_notary=with_notary, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
     try:
         validate(config_dict, notary_mode=with_notary)
     except Exception as e:
@@ -47,7 +44,7 @@ def prepare(conf, with_notary, with_clair, with_trivy, with_chartmuseum):
     prepare_portal(config_dict)
     prepare_log_configs(config_dict)
     prepare_nginx(config_dict)
-    prepare_core(config_dict, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
+    prepare_core(config_dict, with_notary=with_notary, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
     prepare_registry(config_dict)
     prepare_registry_ctl(config_dict)
     prepare_db(config_dict)
@@ -68,14 +65,10 @@ def prepare(conf, with_notary, with_clair, with_trivy, with_chartmuseum):
     if with_notary:
         prepare_notary(config_dict, nginx_confd_dir, SSL_CERT_PATH, SSL_CERT_KEY_PATH)
 
-    if with_clair:
-        prepare_clair(config_dict)
-        prepare_clair_adapter(config_dict)
-
     if with_trivy:
         prepare_trivy_adapter(config_dict)
 
     if with_chartmuseum:
         prepare_chartmuseum(config_dict)
 
-    prepare_docker_compose(config_dict, with_clair, with_trivy, with_notary, with_chartmuseum)
+    prepare_docker_compose(config_dict, with_trivy, with_notary, with_chartmuseum)

make/photon/prepare/g.py

@@ -57,10 +57,8 @@ INTERNAL_NO_PROXY_DN = {
     'jobservice',
     'registry',
     'registryctl',
-    'clair',
     'chartmuseum',
     'notary-server',
     'notary-signer',
-    'clair-adapter',
     'trivy-adapter',
     }

make/photon/prepare/models.py

@@ -19,11 +19,6 @@ class InternalTLS:
         'portal.crt', 'portal.key'
     }
 
-    clair_certs_filename = {
-        'clair_adapter.crt', 'clair_adapter.key',
-        'clair.crt', 'clair.key'
-    }
-
     trivy_certs_filename = {
         'trivy_adapter.crt', 'trivy_adapter.key',
     }
@@ -49,8 +44,6 @@ class InternalTLS:
         self.tls_dir = tls_dir
         if self.enabled:
             self.required_filenames = self.harbor_certs_filename
-            if kwargs.get('with_clair'):
-                self.required_filenames.update(self.clair_certs_filename)
             if kwargs.get('with_notary'):
                 self.required_filenames.update(self.notary_certs_filename)
             if kwargs.get('with_chartmuseum'):

make/photon/prepare/scripts/gencert.sh

@@ -77,27 +77,6 @@ openssl req -new \
 openssl x509 -req -days $DAYS -sha256 -in registryctl.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out registryctl.crt
 
 
-
-# generate clair_adapter key
-openssl req -new \
-        -newkey rsa:4096 -nodes -sha256 -keyout clair_adapter.key \
-        -out clair_adapter.csr \
-        -subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=clair-adapter"
-
-# sign clair_adapter csr with CA certificate and key
-openssl x509 -req -days $DAYS -sha256 -in clair_adapter.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out clair_adapter.crt
-
-
-# generate clair key
-openssl req -new \
-        -newkey rsa:4096 -nodes -sha256 -keyout clair.key \
-        -out clair.csr \
-        -subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=clair"
-
-# sign clair csr with CA certificate and key
-openssl x509 -req -days $DAYS -sha256 -in clair.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out clair.crt
-
-
 # generate trivy_adapter key
 openssl req -new \
         -newkey rsa:4096 -nodes -sha256 -keyout trivy_adapter.key \

make/photon/prepare/templates/clair-adapter/env.jinja

@@ -1,10 +0,0 @@
-SCANNER_LOG_LEVEL={{log_level}}
-SCANNER_CLAIR_URL={{clair_url}}
-SCANNER_CLAIR_DATABASE_URL=postgresql://{{clair_db_username}}:{{clair_db_password}}@{{clair_db_host}}:{{clair_db_port}}/{{clair_db_name}}?sslmode={{clair_db_sslmode}}
-SCANNER_STORE_REDIS_URL={{redis_url_clair}}
-
-{%if internal_tls.enabled %}
-SCANNER_API_SERVER_ADDR=:8443
-SCANNER_API_SERVER_TLS_CERTIFICATE=/etc/harbor/ssl/clair_adapter.crt
-SCANNER_API_SERVER_TLS_KEY=/etc/harbor/ssl/clair_adapter.key
-{% endif %}

make/photon/prepare/templates/clair/clair_env.jinja

@@ -1,3 +0,0 @@
-HTTP_PROXY={{clair_http_proxy}}
-HTTPS_PROXY={{clair_https_proxy}}
-NO_PROXY={{clair_no_proxy}}

make/photon/prepare/templates/clair/config.yaml.jinja

@@ -1,19 +0,0 @@
-clair:
-  database:
-    type: pgsql
-    options:
-      source: postgresql://{{clair_db_username}}:{{clair_db_password}}@{{clair_db_host}}:{{clair_db_port}}/{{clair_db_name}}?sslmode={{clair_db_sslmode}}
-
-      # Number of elements kept in the cache
-      # Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database.
-      cachesize: 16384
-
-  api:
-    # API server port
-    port: 6060
-    healthport: 6061
-
-    # Deadline before an API request will respond with a 503
-    timeout: 300s
-  updater:
-    interval: {{clair_updaters_interval}}h

make/photon/prepare/templates/clair/postgres_env.jinja

@@ -1 +0,0 @@
-POSTGRES_PASSWORD={{clair_db_password}}

make/photon/prepare/templates/clair/postgresql-init.d/README.md

@@ -1,7 +0,0 @@
-This folder used to run some initial sql for clair if needed.
-
-Just put the sql file in this directory and then start the
-clair . 
-
-both .sql and .gz format supported
-

make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja

@@ -61,9 +61,6 @@ services:
 {% endif %}
     networks:
       - harbor
-{% if with_clair %}
-      - harbor-clair
-{% endif %}
     dns_search: .
     depends_on:
       - log
@@ -136,11 +133,6 @@ services:
       harbor-notary:
         aliases:
           - harbor-db
-  {% endif %}
-  {% if with_clair %}
-      harbor-clair:
-        aliases:
-          - harbor-db
   {% endif %}
     dns_search: .
     env_file:
@@ -198,11 +190,6 @@ services:
 {% if with_notary %}
       harbor-notary:
 {% endif %}
-{% if with_clair %}
-      harbor-clair:
-        aliases:
-          - harbor-core
-{% endif %}
 {% if with_chartmuseum %}
       harbor-chartmuseum:
         aliases:
@@ -287,9 +274,6 @@ services:
 {% endif %}
     networks:
       - harbor
-{% if with_clair %}
-      - harbor-clair
-{% endif %}
     dns_search: .
     depends_on:
       - core
@@ -317,11 +301,6 @@ services:
       harbor-chartmuseum:
         aliases:
           - redis
-  {% endif %}
-  {% if with_clair %}
-      harbor-clair:
-        aliases:
-          - redis
   {% endif %}
     dns_search: .
     depends_on:
@@ -468,87 +447,6 @@ services:
         syslog-address: "tcp://127.0.0.1:1514"
         tag: "notary-signer"
 {% endif %}
-{% if with_clair %}
-  clair:
-    networks:
-      - harbor-clair
-    container_name: clair
-    image: goharbor/clair-photon:{{clair_version}}
-    restart: always
-    cap_drop:
-      - ALL
-    cap_add:
-      - DAC_OVERRIDE
-      - SETGID
-      - SETUID
-    cpu_quota: 50000
-    dns_search: .
-    depends_on:
-      - log
-  {% if external_database == False %}
-      - postgresql
-  {% endif %}
-    volumes:
-      - type: bind
-        source: ./common/config/clair/config.yaml
-        target: /etc/clair/config.yaml
-      - type: bind
-        source: ./common/config/shared/trust-certificates
-        target: /harbor_cust_cert
-{%if internal_tls.enabled %}
-      - type: bind
-        source: {{internal_tls.clair_crt_path}}
-        target: /etc/harbor/ssl/clair.crt
-      - type: bind
-        source: {{internal_tls.clair_key_path}}
-        target: /etc/harbor/ssl/clair.key
-{% endif %}
-    logging:
-      driver: "syslog"
-      options:
-        syslog-address: "tcp://127.0.0.1:1514"
-        tag: "clair"
-    env_file:
-      ./common/config/clair/clair_env
-  clair-adapter:
-    networks:
-      - harbor-clair
-    container_name: clair-adapter
-    image: goharbor/clair-adapter-photon:{{clair_adapter_version}}
-    restart: always
-    cap_drop:
-      - ALL
-    cap_add:
-      - DAC_OVERRIDE
-      - SETGID
-      - SETUID
-    cpu_quota: 50000
-    dns_search: .
-    depends_on:
-      - clair
-{% if external_redis == False %}
-      - redis
-{% endif %}
-    volumes:
-      - type: bind
-        source: ./common/config/shared/trust-certificates
-        target: /harbor_cust_cert
-{%if internal_tls.enabled %}
-      - type: bind
-        source: {{internal_tls.clair_adapter_crt_path}}
-        target: /etc/harbor/ssl/clair_adapter.crt
-      - type: bind
-        source: {{internal_tls.clair_adapter_key_path}}
-        target: /etc/harbor/ssl/clair_adapter.key
-{% endif %}
-    logging:
-      driver: "syslog"
-      options:
-        syslog-address: "tcp://127.0.0.1:1514"
-        tag: "clair-adapter"
-    env_file:
-      ./common/config/clair-adapter/env
-{% endif %}
 {% if with_trivy %}
   trivy-adapter:
     container_name: trivy-adapter
@@ -643,10 +541,6 @@ networks:
   notary-sig:
     external: false
 {% endif %}
-{% if with_clair %}
-  harbor-clair:
-    external: false
-{% endif %}
 {% if with_chartmuseum %}
   harbor-chartmuseum:
     external: false

make/photon/prepare/utils/clair.py

@@ -1,43 +0,0 @@
-import os, shutil
-
-from g import templates_dir, config_dir, DEFAULT_UID, DEFAULT_GID
-from .jinja import render_jinja
-from .misc import prepare_dir
-
-clair_template_dir = os.path.join(templates_dir, "clair")
-
-def prepare_clair(config_dict):
-    clair_config_dir = prepare_dir(config_dir, "clair")
-
-    if os.path.exists(os.path.join(clair_config_dir, "postgresql-init.d")):
-        print("Copying offline data file for clair DB")
-        shutil.rmtree(os.path.join(clair_config_dir, "postgresql-init.d"))
-
-    shutil.copytree(os.path.join(clair_template_dir, "postgresql-init.d"), os.path.join(clair_config_dir, "postgresql-init.d"))
-
-    postgres_env_path = os.path.join(clair_config_dir, "postgres_env")
-    postgres_env_template = os.path.join(clair_template_dir, "postgres_env.jinja")
-
-    clair_config_path = os.path.join(clair_config_dir, "config.yaml")
-    clair_config_template = os.path.join(clair_template_dir, "config.yaml.jinja")
-
-    clair_env_path = os.path.join(clair_config_dir, "clair_env")
-    clair_env_template = os.path.join(clair_template_dir, "clair_env.jinja")
-
-    render_jinja(
-        postgres_env_template,
-        postgres_env_path,
-        **config_dict)
-
-    render_jinja(
-        clair_config_template,
-        clair_config_path,
-        uid=DEFAULT_UID,
-        gid=DEFAULT_GID,
-        **config_dict)
-
-    # config http proxy for Clair
-    render_jinja(
-        clair_env_template,
-        clair_env_path,
-        **config_dict)

make/photon/prepare/utils/clair_adapter.py

@@ -1,18 +0,0 @@
-import os
-
-from g import templates_dir, config_dir
-from .jinja import render_jinja
-from .misc import prepare_dir
-
-clair_adapter_template_dir = os.path.join(templates_dir, "clair-adapter")
-
-def prepare_clair_adapter(config_dict):
-    clair_adapter_config_dir = prepare_dir(config_dir, "clair-adapter")
-
-    clair_adapter_env_path = os.path.join(clair_adapter_config_dir, "env")
-    clair_adapter_env_template = os.path.join(clair_adapter_template_dir, "env.jinja")
-
-    render_jinja(
-        clair_adapter_env_template,
-        clair_adapter_env_path,
-        **config_dict)

make/photon/prepare/utils/configs.py

@@ -85,7 +85,7 @@ def parse_versions():
     return versions
 
 
-def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, with_chartmuseum):
+def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseum):
     '''
     :param configs: config_parser object
     :returns: dict of configs
@@ -102,8 +102,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
         'core_local_url': 'http://127.0.0.1:8080',
         'token_service_url': 'http://core:8080/service/token',
         'jobservice_url': 'http://jobservice:8080',
-        'clair_url': 'http://clair:6060',
-        'clair_adapter_url': 'http://clair-adapter:8080',
         'trivy_adapter_url': 'http://trivy-adapter:8080',
         'notary_url': 'http://notary-server:4443',
         'chart_repository_url': 'http://chartmuseum:9999'
@@ -149,15 +147,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
         config_dict['harbor_db_max_idle_conns'] = db_configs.get("max_idle_conns") or default_db_max_idle_conns
         config_dict['harbor_db_max_open_conns'] = db_configs.get("max_open_conns") or default_db_max_open_conns
 
-        if with_clair:
-            # clair db
-            config_dict['clair_db_host'] = 'postgresql'
-            config_dict['clair_db_port'] = 5432
-            config_dict['clair_db_name'] = 'postgres'
-            config_dict['clair_db_username'] = 'postgres'
-            config_dict['clair_db_password'] = db_configs.get("password") or ''
-            config_dict['clair_db_sslmode'] = 'disable'
-
         if with_notary:
             # notary signer
             config_dict['notary_signer_db_host'] = 'postgresql'
@@ -224,12 +213,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
       config_dict[proxy_component + '_https_proxy'] = proxy_config.get('https_proxy') or ''
       config_dict[proxy_component + '_no_proxy'] = ','.join(all_no_proxy)
 
-    # Clair configs, optional
-    clair_configs = configs.get("clair") or {}
-    config_dict['clair_db'] = 'postgres'
-    updaters_interval = clair_configs.get("updaters_interval", None)
-    config_dict['clair_updaters_interval'] = 12 if updaters_interval is None else updaters_interval
-
     # Trivy configs, optional
     trivy_configs = configs.get("trivy") or {}
     config_dict['trivy_github_token'] = trivy_configs.get("github_token") or ''
@@ -292,14 +275,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
         config_dict['harbor_db_max_idle_conns'] = external_db_configs['harbor'].get("max_idle_conns") or default_db_max_idle_conns
         config_dict['harbor_db_max_open_conns'] = external_db_configs['harbor'].get("max_open_conns") or default_db_max_open_conns
 
-        if with_clair:
-            # clair db
-            config_dict['clair_db_host'] = external_db_configs['clair']['host']
-            config_dict['clair_db_port'] = external_db_configs['clair']['port']
-            config_dict['clair_db_name'] = external_db_configs['clair']['db_name']
-            config_dict['clair_db_username'] = external_db_configs['clair']['username']
-            config_dict['clair_db_password'] = external_db_configs['clair']['password']
-            config_dict['clair_db_sslmode'] = external_db_configs['clair']['ssl_mode']
         if with_notary:
             # notary signer
             config_dict['notary_signer_db_host'] = external_db_configs['notary_signer']['host']
@@ -319,7 +294,7 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
         config_dict['external_database'] = False
 
     # update redis configs
-    config_dict.update(get_redis_configs(configs.get("external_redis", None), with_clair, with_trivy))
+    config_dict.update(get_redis_configs(configs.get("external_redis", None), with_trivy))
 
     # auto generated secret string for core
     config_dict['core_secret'] = generate_random_string(16)
@@ -339,7 +314,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
             internal_tls_config['dir'],
             configs['data_volume'],
             with_notary=with_notary,
-            with_clair=with_clair,
             with_trivy=with_trivy,
             with_chartmuseum=with_chartmuseum,
             external_database=config_dict['external_database'])
@@ -361,7 +335,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
         config_dict['core_local_url'] = 'https://core:8443'
         config_dict['token_service_url'] = 'https://core:8443/service/token'
         config_dict['jobservice_url'] = 'https://jobservice:8443'
-        config_dict['clair_adapter_url'] = 'https://clair-adapter:8443'
         config_dict['trivy_adapter_url'] = 'https://trivy-adapter:8443'
         # config_dict['notary_url'] = 'http://notary-server:4443'
         config_dict['chart_repository_url'] = 'https://chartmuseum:9443'
@@ -404,7 +377,7 @@ def get_redis_url_param(redis=None):
     return ""
 
 
-def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
+def get_redis_configs(external_redis=None, with_trivy=True):
     """Returns configs for redis
 
     >>> get_redis_configs()['external_redis']
@@ -413,8 +386,6 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
     'redis://redis:6379/1'
     >>> get_redis_configs()['redis_url_js']
     'redis://redis:6379/2'
-    >>> get_redis_configs()['redis_url_clair']
-    'redis://redis:6379/4'
     >>> get_redis_configs()['trivy_redis_url']
     'redis://redis:6379/5'
 
@@ -433,13 +404,9 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
     'redis://anonymous:pass@localhost:6379/1'
     >>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['redis_url_js']
     'redis://anonymous:pass@localhost:6379/2'
-    >>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['redis_url_clair']
-    'redis://anonymous:pass@localhost:6379/4'
     >>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['trivy_redis_url']
     'redis://anonymous:pass@localhost:6379/5'
 
-    >>> 'redis_url_clair' not in get_redis_configs(with_clair=False)
-    True
     >>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False)
     True
     """
@@ -454,7 +421,6 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
         'registry_db_index': 1,
         'jobservice_db_index': 2,
         'chartmuseum_db_index': 3,
-        'clair_db_index': 4,
         'trivy_db_index': 5,
         'idle_timeout_seconds': 30,
     }
@@ -467,9 +433,6 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
     configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis)
     configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis)
 
-    if with_clair:
-        configs['redis_url_clair'] = get_redis_url(redis['clair_db_index'], redis)
-
     if with_trivy:
         configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis)
 

make/photon/prepare/utils/core.py

@@ -13,7 +13,7 @@ core_conf = os.path.join(config_dir, "core", "app.conf")
 ca_download_dir = os.path.join(data_dir, 'ca_download')
 
 
-def prepare_core(config_dict, with_notary, with_clair, with_trivy, with_chartmuseum):
+def prepare_core(config_dict, with_notary, with_trivy, with_chartmuseum):
     prepare_dir(ca_download_dir, uid=DEFAULT_UID, gid=DEFAULT_GID)
     prepare_dir(core_config_dir)
     # Render Core
@@ -29,7 +29,6 @@ def prepare_core(config_dict, with_notary, with_clair, with_trivy, with_chartmus
         core_conf_env,
         chart_cache_driver=chart_cache_driver,
         with_notary=with_notary,
-        with_clair=with_clair,
         with_trivy=with_trivy,
         with_chartmuseum=with_chartmuseum,
         csrf_key=generate_random_string(32),

make/photon/prepare/utils/docker_compose.py

@@ -8,7 +8,7 @@ docker_compose_template_path = os.path.join(templates_dir, 'docker_compose', 'do
 docker_compose_yml_path = '/compose_location/docker-compose.yml'
 
 # render docker-compose
-def prepare_docker_compose(configs, with_clair, with_trivy, with_notary, with_chartmuseum):
+def prepare_docker_compose(configs, with_trivy, with_notary, with_chartmuseum):
     versions = parse_versions()
     VERSION_TAG = versions.get('VERSION_TAG') or 'dev'
 
@@ -17,8 +17,6 @@ def prepare_docker_compose(configs, with_clair, with_trivy, with_notary, with_ch
         'reg_version': VERSION_TAG,
         'redis_version': VERSION_TAG,
         'notary_version': VERSION_TAG,
-        'clair_version': VERSION_TAG,
-        'clair_adapter_version': VERSION_TAG,
         'trivy_adapter_version': VERSION_TAG,
         'chartmuseum_version': VERSION_TAG,
         'data_volume': configs['data_volume'],
@@ -28,7 +26,6 @@ def prepare_docker_compose(configs, with_clair, with_trivy, with_notary, with_ch
         'external_redis': configs['external_redis'],
         'external_database': configs['external_database'],
         'with_notary': with_notary,
-        'with_clair': with_clair,
         'with_trivy': with_trivy,
         'with_chartmuseum': with_chartmuseum
     }