Merge pull request #13421 from ninjadq/remove_clair

Remove clair
This commit is contained in:
Qian Deng 2020-11-10 13:58:39 +08:00 committed by GitHub
commit 10f0654230
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 21 additions and 540 deletions

View File

@ -76,7 +76,6 @@ REGISTRYSERVER=
REGISTRYPROJECTNAME=goharbor
DEVFLAG=true
NOTARYFLAG=false
CLAIRFLAG=false
TRIVYFLAG=false
HTTPPROXY=
BUILDBIN=false
@ -101,9 +100,7 @@ PREPARE_VERSION_NAME=versions
#versions
REGISTRYVERSION=v2.7.1-patch-2819-2553-redis
NOTARYVERSION=v0.6.1
CLAIRVERSION=v2.1.6
NOTARYMIGRATEVERSION=v3.5.4
CLAIRADAPTERVERSION=v1.1.1
TRIVYVERSION=v0.9.2
TRIVYADAPTERVERSION=v0.14.1
@ -117,11 +114,9 @@ CHARTMUSEUM_SRC_TAG=v0.12.0
REGISTRY_SRC_TAG=v2.7.1
# dependency binaries
CLAIRURL=https://storage.googleapis.com/harbor-builds/bin/clair/release2.0-${CLAIRVERSION}/clair
CHARTURL=https://storage.googleapis.com/harbor-builds/bin/chartmuseum/release-${CHARTMUSEUMVERSION}/chartm
NORARYURL=https://storage.googleapis.com/harbor-builds/bin/notary/release-${NOTARYVERSION}/binary-bundle.tgz
REGISTRYURL=https://storage.googleapis.com/harbor-builds/bin/registry/release-${REGISTRYVERSION}/registry
CLAIR_ADAPTER_DOWNLOAD_URL=https://github.com/goharbor/harbor-scanner-clair/releases/download/$(CLAIRADAPTERVERSION)/harbor-scanner-clair_$(CLAIRADAPTERVERSION:v%=%)_Linux_x86_64.tar.gz
TRIVY_DOWNLOAD_URL=https://github.com/aquasecurity/trivy/releases/download/$(TRIVYVERSION)/trivy_$(TRIVYVERSION:v%=%)_Linux-64bit.tar.gz
TRIVY_ADAPTER_DOWNLOAD_URL=https://github.com/aquasecurity/harbor-scanner-trivy/releases/download/$(TRIVYADAPTERVERSION)/harbor-scanner-trivy_$(TRIVYADAPTERVERSION:v%=%)_Linux_x86_64.tar.gz
@ -129,8 +124,6 @@ define VERSIONS_FOR_PREPARE
VERSION_TAG: $(VERSIONTAG)
REGISTRY_VERSION: $(REGISTRYVERSION)
NOTARY_VERSION: $(NOTARYVERSION)
CLAIR_VERSION: $(CLAIRVERSION)
CLAIR_ADAPTER_VERSION: $(CLAIRADAPTERVERSION)
TRIVY_VERSION: $(TRIVYVERSION)
TRIVY_ADAPTER_VERSION: $(TRIVYADAPTERVERSION)
CHARTMUSEUM_VERSION: $(CHARTMUSEUMVERSION)
@ -210,9 +203,6 @@ PREPARECMD_PARA=--conf $(INSIDE_CONFIGPATH)/$(CONFIGFILE)
ifeq ($(NOTARYFLAG), true)
PREPARECMD_PARA+= --with-notary
endif
ifeq ($(CLAIRFLAG), true)
PREPARECMD_PARA+= --with-clair
endif
ifeq ($(TRIVYFLAG), true)
PREPARECMD_PARA+= --with-trivy
endif
@ -239,14 +229,7 @@ DOCKERIMAGENAME_REGCTL=goharbor/harbor-registryctl
# docker-compose files
DOCKERCOMPOSEFILEPATH=$(MAKEPATH)
DOCKERCOMPOSETPLFILENAME=docker-compose.tpl
DOCKERCOMPOSEFILENAME=docker-compose.yml
DOCKERCOMPOSENOTARYTPLFILENAME=docker-compose.notary.tpl
DOCKERCOMPOSENOTARYFILENAME=docker-compose.notary.yml
DOCKERCOMPOSECLAIRTPLFILENAME=docker-compose.clair.tpl
DOCKERCOMPOSECLAIRFILENAME=docker-compose.clair.yml
DOCKERCOMPOSECHARTMUSEUMTPLFILENAME=docker-compose.chartmuseum.tpl
DOCKERCOMPOSECHARTMUSEUMFILENAME=docker-compose.chartmuseum.yml
SEDCMD=$(shell which sed)
SEDCMDI=$(SEDCMD) -i
@ -297,9 +280,6 @@ DOCKERCOMPOSE_FILE_OPT=-f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
ifeq ($(NOTARYFLAG), true)
DOCKERSAVE_PARA+= goharbor/notary-server-photon:$(VERSIONTAG) goharbor/notary-signer-photon:$(VERSIONTAG)
endif
ifeq ($(CLAIRFLAG), true)
DOCKERSAVE_PARA+= goharbor/clair-photon:$(VERSIONTAG) goharbor/clair-adapter-photon:$(VERSIONTAG)
endif
ifeq ($(TRIVYFLAG), true)
DOCKERSAVE_PARA+= goharbor/trivy-adapter-photon:$(VERSIONTAG)
endif
@ -409,18 +389,18 @@ build:
-e REGISTRYVERSION=$(REGISTRYVERSION) -e REGISTRY_SRC_TAG=$(REGISTRY_SRC_TAG) \
-e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
-e TRIVYVERSION=$(TRIVYVERSION) -e TRIVYADAPTERVERSION=$(TRIVYADAPTERVERSION) \
-e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRADAPTERVERSION=$(CLAIRADAPTERVERSION) -e VERSIONTAG=$(VERSIONTAG) \
-e VERSIONTAG=$(VERSIONTAG) \
-e BUILDBIN=$(BUILDBIN) \
-e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e CHARTMUSEUM_SRC_TAG=$(CHARTMUSEUM_SRC_TAG) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER) \
-e NPM_REGISTRY=$(NPM_REGISTRY) -e BASEIMAGETAG=$(BASEIMAGETAG) -e BASEIMAGENAMESPACE=$(BASEIMAGENAMESPACE) \
-e CLAIRURL=$(CLAIRURL) -e CHARTURL=$(CHARTURL) -e NORARYURL=$(NORARYURL) -e REGISTRYURL=$(REGISTRYURL) -e CLAIR_ADAPTER_DOWNLOAD_URL=$(CLAIR_ADAPTER_DOWNLOAD_URL) \
-e CHARTURL=$(CHARTURL) -e NORARYURL=$(NORARYURL) -e REGISTRYURL=$(REGISTRYURL) \
-e TRIVY_DOWNLOAD_URL=$(TRIVY_DOWNLOAD_URL) -e TRIVY_ADAPTER_DOWNLOAD_URL=$(TRIVY_ADAPTER_DOWNLOAD_URL)
build_standalone_db_migrator: compile_standalone_db_migrator
make -f $(MAKEFILEPATH_PHOTON)/Makefile _build_standalone_db_migrator -e BASEIMAGETAG=$(BASEIMAGETAG) -e VERSIONTAG=$(VERSIONTAG)
build_base_docker:
@for name in chartserver clair clair-adapter trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
@for name in chartserver trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
echo $$name ; \
$(DOCKERBUILD) --pull --no-cache -f $(MAKEFILEPATH_PHOTON)/$$name/Dockerfile.base -t $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) --label base-build-date=$(date +"%Y%m%d") . && \
if [ -n "$(PUSHBASEIMAGE)" ] ; then \
@ -429,7 +409,7 @@ build_base_docker:
done
pull_base_docker:
@for name in chartserver clair clair-adapter trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
@for name in chartserver trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
echo $$name ; \
$(DOCKERPULL) $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) ; \
done

View File

@ -61,11 +61,6 @@ data_volume: /data
# redirect:
# disabled: false
# Clair configuration
clair:
# The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
updaters_interval: 12
# Trivy configuration
#
# Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases.
@ -147,13 +142,6 @@ _version: 2.0.0
# ssl_mode: disable
# max_idle_conns: 2
# max_open_conns: 0
# clair:
# host: clair_db_host
# port: clair_db_port
# db_name: clair_db_name
# username: clair_db_username
# password: clair_db_password
# ssl_mode: disable
# notary_signer:
# host: notary_signer_db_host
# port: notary_signer_db_port
@ -206,7 +194,6 @@ proxy:
components:
- core
- jobservice
- clair
- trivy
# metric:

View File

@ -9,14 +9,13 @@ set +o noglob
usage=$'Please set hostname and other necessary attributes in harbor.yml first. DO NOT use localhost or 127.0.0.1 for hostname, because Harbor needs to be accessed by external clients.
Please set --with-notary if needs enable Notary in Harbor, and set ui_url_protocol/ssl_cert/ssl_cert_key in harbor.yml bacause notary must run under https.
Please set --with-clair if needs enable Clair in Harbor
Please set --with-trivy if needs enable Trivy in Harbor
Please set --with-chartmuseum if needs enable Chartmuseum in Harbor'
item=0
# notary is not enabled by default
with_notary=$false
# clair is not enabled by default
# clair is deprecated
with_clair=$false
# trivy is not enabled by default
with_trivy=$false
@ -43,6 +42,12 @@ while [ $# -gt 0 ]; do
shift || true
done
if [ $with_clair ]
then
error "Clair is deprecated please remove it from installation arguments !!!"
exit 1
fi
workdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd $workdir
@ -71,10 +76,6 @@ if [ $with_notary ]
then
prepare_para="${prepare_para} --with-notary"
fi
if [ $with_clair ]
then
prepare_para="${prepare_para} --with-clair"
fi
if [ $with_trivy ]
then
prepare_para="${prepare_para} --with-trivy"

View File

@ -59,14 +59,6 @@ DOCKERFILEPATH_POSTGRESQL=$(DOCKERFILEPATH)/postgresql
DOCKERFILENAME_POSTGRESQL=Dockerfile
DOCKERIMAGENAME_POSTGRESQL=goharbor/postgresql-photon
DOCKERFILEPATH_CLAIR=$(DOCKERFILEPATH)/clair
DOCKERFILENAME_CLAIR=Dockerfile
DOCKERIMAGENAME_CLAIR=goharbor/clair-photon
DOCKERFILEPATH_CLAIR_ADAPTER=$(DOCKERFILEPATH)/clair-adapter
DOCKERFILENAME_CLAIR_ADAPTER=Dockerfile
DOCKERIMAGENAME_CLAIR_ADAPTER=goharbor/clair-adapter-photon
DOCKERFILEPATH_TRIVY_ADAPTER=$(DOCKERFILEPATH)/trivy-adapter
DOCKERFILENAME_TRIVY_ADAPTER=Dockerfile
DOCKERIMAGENAME_TRIVY_ADAPTER=goharbor/trivy-adapter-photon
@ -137,38 +129,6 @@ _build_log:
$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_LOG)/$(DOCKERFILENAME_LOG) -t $(DOCKERIMAGENAME_LOG):$(VERSIONTAG) .
@echo "Done."
_build_clair:
@if [ "$(CLAIRFLAG)" = "true" ] ; then \
if [ "$(BUILDBIN)" != "true" ] ; then \
rm -rf $(DOCKERFILEPATH_CLAIR)/binary && mkdir -p $(DOCKERFILEPATH_CLAIR)/binary && \
$(call _get_binary, $(CLAIRURL), $(DOCKERFILEPATH_CLAIR)/binary/clair); \
else \
cd $(DOCKERFILEPATH_CLAIR) && $(DOCKERFILEPATH_CLAIR)/builder $(CLAIRVERSION) && cd - ; \
fi ; \
echo "building clair container for photon..." ; \
$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_CLAIR)/$(DOCKERFILENAME_CLAIR) -t $(DOCKERIMAGENAME_CLAIR):$(VERSIONTAG) . ; \
rm -rf $(DOCKERFILEPATH_CLAIR)/binary; \
echo "Done." ; \
fi
_build_clair_adapter:
@if [ "$(CLAIRFLAG)" = "true" ] ; then \
if [ "$(BUILDBIN)" != "true" ] ; then \
rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary && mkdir -p $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary && \
$(call _extract_archive, $(CLAIR_ADAPTER_DOWNLOAD_URL), $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/) && \
mv $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/scanner-clair $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/harbor-scanner-clair; \
else \
cd $(DOCKERFILEPATH_CLAIR_ADAPTER) && $(DOCKERFILEPATH_CLAIR_ADAPTER)/builder.sh $(CLAIRADAPTERVERSION) && cd - ; \
fi ; \
echo "Building Clair adapter container for photon..." ; \
$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) \
--build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) \
-f $(DOCKERFILEPATH_CLAIR_ADAPTER)/$(DOCKERFILENAME_CLAIR_ADAPTER) \
-t $(DOCKERIMAGENAME_CLAIR_ADAPTER):$(VERSIONTAG) . ; \
rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary; \
echo "Done." ; \
fi
_build_trivy_adapter:
@if [ "$(TRIVYFLAG)" = "true" ] ; then \
rm -rf $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary && mkdir -p $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary ; \
@ -263,7 +223,7 @@ define _get_binary
$(CURL) --connect-timeout 30 -f -k -L $1 -o $2 || exit 1
endef
build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_clair_adapter _build_trivy_adapter _build_redis _build_chart_server
build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_trivy_adapter _build_redis _build_chart_server
cleanimage:
@echo "cleaning image for photon..."

View File

@ -1,18 +0,0 @@
ARG harbor_base_image_version
ARG harbor_base_namespace
FROM ${harbor_base_namespace}/harbor-clair-adapter-base:${harbor_base_image_version}
COPY ./make/photon/common/install_cert.sh /home/clair-adapter
COPY ./make/photon/clair-adapter/entrypoint.sh /home/clair-adapter
COPY ./make/photon/clair-adapter/binary/harbor-scanner-clair /clair-adapter/clair-adapter
RUN chown -R clair-adapter:clair-adapter /etc/pki/tls/certs \
&& chown -R clair-adapter:clair-adapter /clair-adapter && chmod u+x /clair-adapter/clair-adapter \
&& chown clair-adapter:clair-adapter /home/clair-adapter/entrypoint.sh && chmod u+x /home/clair-adapter/entrypoint.sh \
&& chown clair-adapter:clair-adapter /home/clair-adapter/install_cert.sh && chmod u+x /home/clair-adapter/install_cert.sh
HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS http://127.0.0.1:8080/probe/healthy || curl -k -sS https://127.0.0.1:8443/probe/healthy || exit 1
USER clair-adapter
ENTRYPOINT ["/home/clair-adapter/entrypoint.sh"]

View File

@ -1,7 +0,0 @@
FROM photon:2.0
RUN tdnf install -y shadow >> /dev/null \
&& tdnf clean all \
&& mkdir /clair-adapter/ \
&& groupadd -r -g 10000 clair-adapter \
&& useradd --no-log-init -m -r -g 10000 -u 10000 clair-adapter

View File

@ -1,11 +0,0 @@
FROM golang:1.14.7
ARG VERSION
ARG COMMIT
ADD . /go/src/github.com/goharbor/harbor-scanner-clair/
WORKDIR /go/src/github.com/goharbor/harbor-scanner-clair/
RUN export GOFLAGS=-mod=vendor GOOS=linux GO111MODULE=on CGO_ENABLED=0 && \
go build -ldflags "-X main.version=$VERSION -X main.commit=$COMMIT -X main.date=`date -u --iso-8601=seconds`" \
-o harbor-scanner-clair cmd/harbor-scanner-clair/main.go

View File

@ -1,39 +0,0 @@
#!/bin/bash
set +e
if [ -z $1 ]; then
error "Please set the 'version' variable"
exit 1
fi
VERSION="$1"
set -e
# the temp folder to store binary file...
mkdir -p binary
rm -rf binary/harbor-scanner-clair || true
cd $(dirname $0)
cur=$PWD
# The temporary directory to clone Clair adapter source code
TEMP=$(mktemp -d ${TMPDIR-/tmp}/clair-adapter.XXXXXX)
git clone https://github.com/goharbor/harbor-scanner-clair.git $TEMP
cd $TEMP; git checkout $VERSION; export COMMIT=$(git rev-list -1 HEAD); cd -
echo "Building Clair adapter binary based on golang:1.14.7..."
cp Dockerfile.binary $TEMP
docker build --build-arg VERSION=${VERSION} --build-arg COMMIT=${COMMIT} -f $TEMP/Dockerfile.binary -t clair-adapter-golang $TEMP
echo "Copying Clair adapter binary from the container to the local directory..."
ID=$(docker create clair-adapter-golang)
docker cp $ID:/go/src/github.com/goharbor/harbor-scanner-clair/harbor-scanner-clair binary
docker rm -f $ID
docker rmi -f clair-adapter-golang
echo "Building Clair adapter binary finished successfully"
cd $cur
rm -rf $TEMP

View File

@ -1,7 +0,0 @@
#!/bin/sh
set -e
/home/clair-adapter/install_cert.sh
/clair-adapter/clair-adapter

View File

@ -1,22 +0,0 @@
ARG harbor_base_image_version
ARG harbor_base_namespace
FROM ${harbor_base_namespace}/harbor-clair-base:${harbor_base_image_version}
COPY ./make/photon/clair/binary/clair /home/clair/
COPY ./make/photon/clair/docker-entrypoint.sh /home/clair/
COPY ./make/photon/clair/dumb-init /home/clair/
COPY ./make/photon/common/install_cert.sh /home/clair/
VOLUME /config
RUN chown -R clair:clair /etc/pki/tls/certs && chown -R clair:clair /home/clair \
&& chmod u+x /home/clair/clair \
&& chmod u+x /home/clair/docker-entrypoint.sh \
&& chmod u+x /home/clair/install_cert.sh \
&& chmod +x /home/clair/dumb-init
HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS 127.0.0.1:6061/health || exit 1
WORKDIR /home/clair
USER clair
ENTRYPOINT ["./docker-entrypoint.sh"]

View File

@ -1,6 +0,0 @@
FROM photon:2.0
RUN tdnf install -y git shadow rpm xz python-xml >>/dev/null\
&& tdnf clean all \
&& groupadd -r -g 10000 clair \
&& useradd --no-log-init -m -g 10000 -u 10000 clair

View File

@ -1,7 +0,0 @@
FROM golang:1.14.7
ADD . /go/src/github.com/quay/clair/
WORKDIR /go/src/github.com/quay/clair/
RUN export CLAIR_VERSION=$(git describe --tag --always --dirty) GO111MODULE=on && \
go build -ldflags "-X github.com/quay/clair/pkg/version.Version=$CLAIR_VERSION" ./cmd/clair

View File

@ -1,38 +0,0 @@
#!/bin/bash
set +e
if [ -z $1 ]; then
error "Please set the 'version' variable"
exit 1
fi
VERSION="$1"
set -e
# the temp folder to store binary file...
mkdir -p binary
rm -rf binary/clair || true
cd `dirname $0`
cur=$PWD
# the temp folder to store distribution source code...
TEMP=`mktemp -d /$TMPDIR/clair.XXXXXX`
git clone -b $VERSION --single-branch https://github.com/quay/clair.git $TEMP
echo 'build the clair binary bases on the golang:1.14.7'
cp Dockerfile.binary $TEMP
docker build -f $TEMP/Dockerfile.binary -t clair-golang $TEMP
echo 'copy the clair binary to local...'
ID=$(docker create clair-golang)
docker cp $ID:/go/src/github.com/quay/clair/clair binary
docker rm -f $ID
docker rmi -f clair-golang
echo "Build clair binary success, then to build photon image..."
cd $cur
rm -rf $TEMP

View File

@ -1,7 +0,0 @@
#!/bin/bash
set -e
/home/clair/install_cert.sh
/home/clair/dumb-init -- /home/clair/clair -config /etc/clair/config.yaml $*
set +e

Binary file not shown.

View File

@ -15,8 +15,6 @@ from utils.registry_ctl import prepare_registry_ctl
from utils.core import prepare_core
from utils.notary import prepare_notary
from utils.log import prepare_log_configs
from utils.clair import prepare_clair
from utils.clair_adapter import prepare_clair_adapter
from utils.chart import prepare_chartmuseum
from utils.docker_compose import prepare_docker_compose
from utils.nginx import prepare_nginx, nginx_confd_dir
@ -30,13 +28,12 @@ old_private_key_pem_path, old_crt_path)
@click.command()
@click.option('--conf', default=input_config_path, help="the path of Harbor configuration file")
@click.option('--with-notary', is_flag=True, help="the Harbor instance is to be deployed with notary")
@click.option('--with-clair', is_flag=True, help="the Harbor instance is to be deployed with clair")
@click.option('--with-trivy', is_flag=True, help="the Harbor instance is to be deployed with Trivy")
@click.option('--with-chartmuseum', is_flag=True, help="the Harbor instance is to be deployed with chart repository supporting")
def prepare(conf, with_notary, with_clair, with_trivy, with_chartmuseum):
def prepare(conf, with_notary, with_trivy, with_chartmuseum):
delfile(config_dir)
config_dict = parse_yaml_config(conf, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
config_dict = parse_yaml_config(conf, with_notary=with_notary, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
try:
validate(config_dict, notary_mode=with_notary)
except Exception as e:
@ -47,7 +44,7 @@ def prepare(conf, with_notary, with_clair, with_trivy, with_chartmuseum):
prepare_portal(config_dict)
prepare_log_configs(config_dict)
prepare_nginx(config_dict)
prepare_core(config_dict, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
prepare_core(config_dict, with_notary=with_notary, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
prepare_registry(config_dict)
prepare_registry_ctl(config_dict)
prepare_db(config_dict)
@ -68,14 +65,10 @@ def prepare(conf, with_notary, with_clair, with_trivy, with_chartmuseum):
if with_notary:
prepare_notary(config_dict, nginx_confd_dir, SSL_CERT_PATH, SSL_CERT_KEY_PATH)
if with_clair:
prepare_clair(config_dict)
prepare_clair_adapter(config_dict)
if with_trivy:
prepare_trivy_adapter(config_dict)
if with_chartmuseum:
prepare_chartmuseum(config_dict)
prepare_docker_compose(config_dict, with_clair, with_trivy, with_notary, with_chartmuseum)
prepare_docker_compose(config_dict, with_trivy, with_notary, with_chartmuseum)

View File

@ -57,10 +57,8 @@ INTERNAL_NO_PROXY_DN = {
'jobservice',
'registry',
'registryctl',
'clair',
'chartmuseum',
'notary-server',
'notary-signer',
'clair-adapter',
'trivy-adapter',
}

View File

@ -19,11 +19,6 @@ class InternalTLS:
'portal.crt', 'portal.key'
}
clair_certs_filename = {
'clair_adapter.crt', 'clair_adapter.key',
'clair.crt', 'clair.key'
}
trivy_certs_filename = {
'trivy_adapter.crt', 'trivy_adapter.key',
}
@ -49,8 +44,6 @@ class InternalTLS:
self.tls_dir = tls_dir
if self.enabled:
self.required_filenames = self.harbor_certs_filename
if kwargs.get('with_clair'):
self.required_filenames.update(self.clair_certs_filename)
if kwargs.get('with_notary'):
self.required_filenames.update(self.notary_certs_filename)
if kwargs.get('with_chartmuseum'):

View File

@ -77,27 +77,6 @@ openssl req -new \
openssl x509 -req -days $DAYS -sha256 -in registryctl.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out registryctl.crt
# generate clair_adapter key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout clair_adapter.key \
-out clair_adapter.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=clair-adapter"
# sign clair_adapter csr with CA certificate and key
openssl x509 -req -days $DAYS -sha256 -in clair_adapter.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out clair_adapter.crt
# generate clair key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout clair.key \
-out clair.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=clair"
# sign clair csr with CA certificate and key
openssl x509 -req -days $DAYS -sha256 -in clair.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out clair.crt
# generate trivy_adapter key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout trivy_adapter.key \

View File

@ -1,10 +0,0 @@
SCANNER_LOG_LEVEL={{log_level}}
SCANNER_CLAIR_URL={{clair_url}}
SCANNER_CLAIR_DATABASE_URL=postgresql://{{clair_db_username}}:{{clair_db_password}}@{{clair_db_host}}:{{clair_db_port}}/{{clair_db_name}}?sslmode={{clair_db_sslmode}}
SCANNER_STORE_REDIS_URL={{redis_url_clair}}
{%if internal_tls.enabled %}
SCANNER_API_SERVER_ADDR=:8443
SCANNER_API_SERVER_TLS_CERTIFICATE=/etc/harbor/ssl/clair_adapter.crt
SCANNER_API_SERVER_TLS_KEY=/etc/harbor/ssl/clair_adapter.key
{% endif %}

View File

@ -1,3 +0,0 @@
HTTP_PROXY={{clair_http_proxy}}
HTTPS_PROXY={{clair_https_proxy}}
NO_PROXY={{clair_no_proxy}}

View File

@ -1,19 +0,0 @@
clair:
database:
type: pgsql
options:
source: postgresql://{{clair_db_username}}:{{clair_db_password}}@{{clair_db_host}}:{{clair_db_port}}/{{clair_db_name}}?sslmode={{clair_db_sslmode}}
# Number of elements kept in the cache
# Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database.
cachesize: 16384
api:
# API server port
port: 6060
healthport: 6061
# Deadline before an API request will respond with a 503
timeout: 300s
updater:
interval: {{clair_updaters_interval}}h

View File

@ -1 +0,0 @@
POSTGRES_PASSWORD={{clair_db_password}}

View File

@ -1,7 +0,0 @@
This folder used to run some initial sql for clair if needed.
Just put the sql file in this directory and then start the
clair .
both .sql and .gz format supported

View File

@ -61,9 +61,6 @@ services:
{% endif %}
networks:
- harbor
{% if with_clair %}
- harbor-clair
{% endif %}
dns_search: .
depends_on:
- log
@ -136,11 +133,6 @@ services:
harbor-notary:
aliases:
- harbor-db
{% endif %}
{% if with_clair %}
harbor-clair:
aliases:
- harbor-db
{% endif %}
dns_search: .
env_file:
@ -198,11 +190,6 @@ services:
{% if with_notary %}
harbor-notary:
{% endif %}
{% if with_clair %}
harbor-clair:
aliases:
- harbor-core
{% endif %}
{% if with_chartmuseum %}
harbor-chartmuseum:
aliases:
@ -287,9 +274,6 @@ services:
{% endif %}
networks:
- harbor
{% if with_clair %}
- harbor-clair
{% endif %}
dns_search: .
depends_on:
- core
@ -317,11 +301,6 @@ services:
harbor-chartmuseum:
aliases:
- redis
{% endif %}
{% if with_clair %}
harbor-clair:
aliases:
- redis
{% endif %}
dns_search: .
depends_on:
@ -468,87 +447,6 @@ services:
syslog-address: "tcp://127.0.0.1:1514"
tag: "notary-signer"
{% endif %}
{% if with_clair %}
clair:
networks:
- harbor-clair
container_name: clair
image: goharbor/clair-photon:{{clair_version}}
restart: always
cap_drop:
- ALL
cap_add:
- DAC_OVERRIDE
- SETGID
- SETUID
cpu_quota: 50000
dns_search: .
depends_on:
- log
{% if external_database == False %}
- postgresql
{% endif %}
volumes:
- type: bind
source: ./common/config/clair/config.yaml
target: /etc/clair/config.yaml
- type: bind
source: ./common/config/shared/trust-certificates
target: /harbor_cust_cert
{%if internal_tls.enabled %}
- type: bind
source: {{internal_tls.clair_crt_path}}
target: /etc/harbor/ssl/clair.crt
- type: bind
source: {{internal_tls.clair_key_path}}
target: /etc/harbor/ssl/clair.key
{% endif %}
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "clair"
env_file:
./common/config/clair/clair_env
clair-adapter:
networks:
- harbor-clair
container_name: clair-adapter
image: goharbor/clair-adapter-photon:{{clair_adapter_version}}
restart: always
cap_drop:
- ALL
cap_add:
- DAC_OVERRIDE
- SETGID
- SETUID
cpu_quota: 50000
dns_search: .
depends_on:
- clair
{% if external_redis == False %}
- redis
{% endif %}
volumes:
- type: bind
source: ./common/config/shared/trust-certificates
target: /harbor_cust_cert
{%if internal_tls.enabled %}
- type: bind
source: {{internal_tls.clair_adapter_crt_path}}
target: /etc/harbor/ssl/clair_adapter.crt
- type: bind
source: {{internal_tls.clair_adapter_key_path}}
target: /etc/harbor/ssl/clair_adapter.key
{% endif %}
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "clair-adapter"
env_file:
./common/config/clair-adapter/env
{% endif %}
{% if with_trivy %}
trivy-adapter:
container_name: trivy-adapter
@ -643,10 +541,6 @@ networks:
notary-sig:
external: false
{% endif %}
{% if with_clair %}
harbor-clair:
external: false
{% endif %}
{% if with_chartmuseum %}
harbor-chartmuseum:
external: false

View File

@ -1,43 +0,0 @@
import os, shutil
from g import templates_dir, config_dir, DEFAULT_UID, DEFAULT_GID
from .jinja import render_jinja
from .misc import prepare_dir
clair_template_dir = os.path.join(templates_dir, "clair")
def prepare_clair(config_dict):
clair_config_dir = prepare_dir(config_dir, "clair")
if os.path.exists(os.path.join(clair_config_dir, "postgresql-init.d")):
print("Copying offline data file for clair DB")
shutil.rmtree(os.path.join(clair_config_dir, "postgresql-init.d"))
shutil.copytree(os.path.join(clair_template_dir, "postgresql-init.d"), os.path.join(clair_config_dir, "postgresql-init.d"))
postgres_env_path = os.path.join(clair_config_dir, "postgres_env")
postgres_env_template = os.path.join(clair_template_dir, "postgres_env.jinja")
clair_config_path = os.path.join(clair_config_dir, "config.yaml")
clair_config_template = os.path.join(clair_template_dir, "config.yaml.jinja")
clair_env_path = os.path.join(clair_config_dir, "clair_env")
clair_env_template = os.path.join(clair_template_dir, "clair_env.jinja")
render_jinja(
postgres_env_template,
postgres_env_path,
**config_dict)
render_jinja(
clair_config_template,
clair_config_path,
uid=DEFAULT_UID,
gid=DEFAULT_GID,
**config_dict)
# config http proxy for Clair
render_jinja(
clair_env_template,
clair_env_path,
**config_dict)

View File

@ -1,18 +0,0 @@
import os
from g import templates_dir, config_dir
from .jinja import render_jinja
from .misc import prepare_dir
clair_adapter_template_dir = os.path.join(templates_dir, "clair-adapter")
def prepare_clair_adapter(config_dict):
clair_adapter_config_dir = prepare_dir(config_dir, "clair-adapter")
clair_adapter_env_path = os.path.join(clair_adapter_config_dir, "env")
clair_adapter_env_template = os.path.join(clair_adapter_template_dir, "env.jinja")
render_jinja(
clair_adapter_env_template,
clair_adapter_env_path,
**config_dict)

View File

@ -85,7 +85,7 @@ def parse_versions():
return versions
def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, with_chartmuseum):
def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseum):
'''
:param configs: config_parser object
:returns: dict of configs
@ -102,8 +102,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
'core_local_url': 'http://127.0.0.1:8080',
'token_service_url': 'http://core:8080/service/token',
'jobservice_url': 'http://jobservice:8080',
'clair_url': 'http://clair:6060',
'clair_adapter_url': 'http://clair-adapter:8080',
'trivy_adapter_url': 'http://trivy-adapter:8080',
'notary_url': 'http://notary-server:4443',
'chart_repository_url': 'http://chartmuseum:9999'
@ -149,15 +147,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
config_dict['harbor_db_max_idle_conns'] = db_configs.get("max_idle_conns") or default_db_max_idle_conns
config_dict['harbor_db_max_open_conns'] = db_configs.get("max_open_conns") or default_db_max_open_conns
if with_clair:
# clair db
config_dict['clair_db_host'] = 'postgresql'
config_dict['clair_db_port'] = 5432
config_dict['clair_db_name'] = 'postgres'
config_dict['clair_db_username'] = 'postgres'
config_dict['clair_db_password'] = db_configs.get("password") or ''
config_dict['clair_db_sslmode'] = 'disable'
if with_notary:
# notary signer
config_dict['notary_signer_db_host'] = 'postgresql'
@ -224,12 +213,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
config_dict[proxy_component + '_https_proxy'] = proxy_config.get('https_proxy') or ''
config_dict[proxy_component + '_no_proxy'] = ','.join(all_no_proxy)
# Clair configs, optional
clair_configs = configs.get("clair") or {}
config_dict['clair_db'] = 'postgres'
updaters_interval = clair_configs.get("updaters_interval", None)
config_dict['clair_updaters_interval'] = 12 if updaters_interval is None else updaters_interval
# Trivy configs, optional
trivy_configs = configs.get("trivy") or {}
config_dict['trivy_github_token'] = trivy_configs.get("github_token") or ''
@ -292,14 +275,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
config_dict['harbor_db_max_idle_conns'] = external_db_configs['harbor'].get("max_idle_conns") or default_db_max_idle_conns
config_dict['harbor_db_max_open_conns'] = external_db_configs['harbor'].get("max_open_conns") or default_db_max_open_conns
if with_clair:
# clair db
config_dict['clair_db_host'] = external_db_configs['clair']['host']
config_dict['clair_db_port'] = external_db_configs['clair']['port']
config_dict['clair_db_name'] = external_db_configs['clair']['db_name']
config_dict['clair_db_username'] = external_db_configs['clair']['username']
config_dict['clair_db_password'] = external_db_configs['clair']['password']
config_dict['clair_db_sslmode'] = external_db_configs['clair']['ssl_mode']
if with_notary:
# notary signer
config_dict['notary_signer_db_host'] = external_db_configs['notary_signer']['host']
@ -319,7 +294,7 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
config_dict['external_database'] = False
# update redis configs
config_dict.update(get_redis_configs(configs.get("external_redis", None), with_clair, with_trivy))
config_dict.update(get_redis_configs(configs.get("external_redis", None), with_trivy))
# auto generated secret string for core
config_dict['core_secret'] = generate_random_string(16)
@ -339,7 +314,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
internal_tls_config['dir'],
configs['data_volume'],
with_notary=with_notary,
with_clair=with_clair,
with_trivy=with_trivy,
with_chartmuseum=with_chartmuseum,
external_database=config_dict['external_database'])
@ -361,7 +335,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
config_dict['core_local_url'] = 'https://core:8443'
config_dict['token_service_url'] = 'https://core:8443/service/token'
config_dict['jobservice_url'] = 'https://jobservice:8443'
config_dict['clair_adapter_url'] = 'https://clair-adapter:8443'
config_dict['trivy_adapter_url'] = 'https://trivy-adapter:8443'
# config_dict['notary_url'] = 'http://notary-server:4443'
config_dict['chart_repository_url'] = 'https://chartmuseum:9443'
@ -404,7 +377,7 @@ def get_redis_url_param(redis=None):
return ""
def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
def get_redis_configs(external_redis=None, with_trivy=True):
"""Returns configs for redis
>>> get_redis_configs()['external_redis']
@ -413,8 +386,6 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
'redis://redis:6379/1'
>>> get_redis_configs()['redis_url_js']
'redis://redis:6379/2'
>>> get_redis_configs()['redis_url_clair']
'redis://redis:6379/4'
>>> get_redis_configs()['trivy_redis_url']
'redis://redis:6379/5'
@ -433,13 +404,9 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
'redis://anonymous:pass@localhost:6379/1'
>>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['redis_url_js']
'redis://anonymous:pass@localhost:6379/2'
>>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['redis_url_clair']
'redis://anonymous:pass@localhost:6379/4'
>>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['trivy_redis_url']
'redis://anonymous:pass@localhost:6379/5'
>>> 'redis_url_clair' not in get_redis_configs(with_clair=False)
True
>>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False)
True
"""
@ -454,7 +421,6 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
'registry_db_index': 1,
'jobservice_db_index': 2,
'chartmuseum_db_index': 3,
'clair_db_index': 4,
'trivy_db_index': 5,
'idle_timeout_seconds': 30,
}
@ -467,9 +433,6 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis)
configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis)
if with_clair:
configs['redis_url_clair'] = get_redis_url(redis['clair_db_index'], redis)
if with_trivy:
configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis)

View File

@ -13,7 +13,7 @@ core_conf = os.path.join(config_dir, "core", "app.conf")
ca_download_dir = os.path.join(data_dir, 'ca_download')
def prepare_core(config_dict, with_notary, with_clair, with_trivy, with_chartmuseum):
def prepare_core(config_dict, with_notary, with_trivy, with_chartmuseum):
prepare_dir(ca_download_dir, uid=DEFAULT_UID, gid=DEFAULT_GID)
prepare_dir(core_config_dir)
# Render Core
@ -29,7 +29,6 @@ def prepare_core(config_dict, with_notary, with_clair, with_trivy, with_chartmus
core_conf_env,
chart_cache_driver=chart_cache_driver,
with_notary=with_notary,
with_clair=with_clair,
with_trivy=with_trivy,
with_chartmuseum=with_chartmuseum,
csrf_key=generate_random_string(32),

View File

@ -8,7 +8,7 @@ docker_compose_template_path = os.path.join(templates_dir, 'docker_compose', 'do
docker_compose_yml_path = '/compose_location/docker-compose.yml'
# render docker-compose
def prepare_docker_compose(configs, with_clair, with_trivy, with_notary, with_chartmuseum):
def prepare_docker_compose(configs, with_trivy, with_notary, with_chartmuseum):
versions = parse_versions()
VERSION_TAG = versions.get('VERSION_TAG') or 'dev'
@ -17,8 +17,6 @@ def prepare_docker_compose(configs, with_clair, with_trivy, with_notary, with_ch
'reg_version': VERSION_TAG,
'redis_version': VERSION_TAG,
'notary_version': VERSION_TAG,
'clair_version': VERSION_TAG,
'clair_adapter_version': VERSION_TAG,
'trivy_adapter_version': VERSION_TAG,
'chartmuseum_version': VERSION_TAG,
'data_volume': configs['data_volume'],
@ -28,7 +26,6 @@ def prepare_docker_compose(configs, with_clair, with_trivy, with_notary, with_ch
'external_redis': configs['external_redis'],
'external_database': configs['external_database'],
'with_notary': with_notary,
'with_clair': with_clair,
'with_trivy': with_trivy,
'with_chartmuseum': with_chartmuseum
}