From 28e0c0693b0161bc1c253ad195ad06feb5e8a090 Mon Sep 17 00:00:00 2001 From: He Weiwei Date: Sat, 26 Oct 2019 17:25:36 +0000 Subject: [PATCH] Upgrade clair adapter to v1.0.0 1. Upgrade clair adapter to v1.0.0. 2. Make the clair adapter which installed by harbor immutable and using internal registry address. 3. Add support to build clair adapter image from binary. 4. Switch to ScannerPull action when make authorization for the scan request. Signed-off-by: He Weiwei --- Makefile | 6 +++--- make/photon/Makefile | 13 +++++++++++-- make/photon/clair-adapter/Dockerfile | 2 +- make/photon/clair-adapter/builder | 2 +- make/photon/prepare/utils/docker_compose.py | 2 +- make/photon/registry/builder | 4 ++-- src/core/main.go | 11 ++++++----- src/pkg/scan/api/scan/base_controller.go | 4 ++-- src/pkg/scan/api/scan/base_controller_test.go | 2 +- tests/hostcfg.sh | 4 ---- 10 files changed, 28 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 2ee4469b2..7966618cc 100644 --- a/Makefile +++ b/Makefile @@ -106,7 +106,7 @@ CLAIRDBVERSION=$(VERSIONTAG) MIGRATORVERSION=$(VERSIONTAG) REDISVERSION=$(VERSIONTAG) NOTARYMIGRATEVERSION=v3.5.4 -CLAIRADAPTERVERSION=c7db8b15 +CLAIRADAPTERVERSION=v1.0.0 # version of chartmuseum CHARTMUSEUMVERSION=v0.9.0 @@ -308,8 +308,8 @@ prepare: update_prepare_version @$(MAKEPATH)/$(PREPARECMD) $(PREPARECMD_PARA) build: - make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) \ - -e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \ + make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) -e GOBUILDIMAGE=$(GOBUILDIMAGE) \ + -e REGISTRYVERSION=$(REGISTRYVERSION) -e REGISTRY_SRC_TAG=$(REGISTRY_SRC_TAG) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \ -e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRADAPTERVERSION=$(CLAIRADAPTERVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \ -e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \ -e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER) \ diff --git a/make/photon/Makefile b/make/photon/Makefile index 4deb9b647..73be3c706 100644 --- a/make/photon/Makefile +++ b/make/photon/Makefile @@ -146,9 +146,14 @@ _build_clair: fi _build_clair_adapter: - # TODO: add support to fetch clair adapter binary from google storage ranther than build from source @if [ "$(CLAIRFLAG)" = "true" ] ; then \ - cd $(DOCKERFILEPATH_CLAIR_ADAPTER) && $(DOCKERFILEPATH_CLAIR_ADAPTER)/builder $(CLAIRADAPTERVERSION) && cd - ; \ + if [ "$(BUILDBIN)" != "true" ] ; then \ + rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary && mkdir -p $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary && \ + $(call _extract_archive, https://github.com/goharbor/harbor-scanner-clair/releases/download/$(CLAIRADAPTERVERSION)/harbor-scanner-clair_$(CLAIRADAPTERVERSION:v%=%)_Linux_x86_64.tar.gz, $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/) && \ + mv $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/scanner-clair $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/harbor-scanner-clair; \ + else \ + cd $(DOCKERFILEPATH_CLAIR_ADAPTER) && $(DOCKERFILEPATH_CLAIR_ADAPTER)/builder $(CLAIRADAPTERVERSION) && cd - ; \ + fi ; \ echo "building clair adapter container for photon..." ; \ $(DOCKERBUILD) -f $(DOCKERFILEPATH_CLAIR_ADAPTER)/$(DOCKERFILENAME_CLAIR_ADAPTER) -t $(DOCKERIMAGENAME_CLAIR_ADAPTER):$(CLAIRADAPTERVERSION)-$(VERSIONTAG) . ; \ rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary; \ @@ -219,6 +224,10 @@ _build_migrator: echo "Done."; \ fi +define _extract_archive + $(WGET) --timeout 30 --no-check-certificate -O- $1 | tar xvz -C $2 +endef + define _get_binary $(WGET) --timeout 30 --no-check-certificate $1 -O $2 endef diff --git a/make/photon/clair-adapter/Dockerfile b/make/photon/clair-adapter/Dockerfile index 1b21ec70d..c3f1a1ded 100644 --- a/make/photon/clair-adapter/Dockerfile +++ b/make/photon/clair-adapter/Dockerfile @@ -13,7 +13,7 @@ RUN chown -R 10000:10000 /clair-adapter \ EXPOSE 8080 -HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS 127.0.0.1:8080/healthy || exit 1 +HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS 127.0.0.1:8080/probe/healthy || exit 1 USER clair-adapter diff --git a/make/photon/clair-adapter/builder b/make/photon/clair-adapter/builder index 82473d855..50acd7c3b 100755 --- a/make/photon/clair-adapter/builder +++ b/make/photon/clair-adapter/builder @@ -20,7 +20,7 @@ cur=$PWD # the temp folder to store distribution source code... TEMP=`mktemp -d ${TMPDIR-/tmp}/clair-adapter.XXXXXX` -git clone https://github.com/danielpacak/harbor-scanner-clair.git $TEMP +git clone https://github.com/goharbor/harbor-scanner-clair.git $TEMP cd $TEMP; git checkout $VERSION; cd - echo 'build the clair adapter binary bases on the golang:1.12.12' diff --git a/make/photon/prepare/utils/docker_compose.py b/make/photon/prepare/utils/docker_compose.py index 05a8f507f..7716b7e39 100644 --- a/make/photon/prepare/utils/docker_compose.py +++ b/make/photon/prepare/utils/docker_compose.py @@ -14,7 +14,7 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum): REGISTRY_VERSION = versions.get('REGISTRY_VERSION') or 'v2.7.1-patch-2819-2553' NOTARY_VERSION = versions.get('NOTARY_VERSION') or 'v0.6.1' CLAIR_VERSION = versions.get('CLAIR_VERSION') or 'v2.0.9' - CLAIR_ADAPTER_VERSION = versions.get('CLAIR_ADAPTER_VERSION') or '' + CLAIR_ADAPTER_VERSION = versions.get('CLAIR_ADAPTER_VERSION') or 'v1.0.0' CHARTMUSEUM_VERSION = versions.get('CHARTMUSEUM_VERSION') or 'v0.9.0' rendering_variables = { diff --git a/make/photon/registry/builder b/make/photon/registry/builder index eaacc1ba2..2a0333875 100755 --- a/make/photon/registry/builder +++ b/make/photon/registry/builder @@ -19,7 +19,7 @@ cd `dirname $0` cur=$PWD # the temp folder to store distribution source code... -TEMP=`mktemp -d /$TMPDIR/distribution.XXXXXX` +TEMP=`mktemp -d ${TMPDIR-/tmp}/distribution.XXXXXX` git clone -b $VERSION https://github.com/docker/distribution.git $TEMP # add patch 2879 @@ -35,7 +35,7 @@ docker build -f $TEMP/Dockerfile.binary -t registry-golang $TEMP echo 'copy the registry binary to local...' ID=$(docker create registry-golang) -docker cp $ID:/go/src/github.com/docker/distribution/bin binary +docker cp $ID:/go/src/github.com/docker/distribution/bin/registry binary/registry docker rm -f $ID docker rmi -f registry-golang diff --git a/src/core/main.go b/src/core/main.go index 244fe0c29..2d32c2d57 100755 --- a/src/core/main.go +++ b/src/core/main.go @@ -219,11 +219,12 @@ func main() { // TODO: change to be internal adapter reg := &scanner.Registration{ - Name: "Clair", - Description: "The clair scanner adapter", - URL: config.ClairAdapterEndpoint(), - Disabled: false, - IsDefault: true, + Name: "Clair", + Description: "The clair scanner adapter", + URL: config.ClairAdapterEndpoint(), + IsDefault: true, + UseInternalAddr: true, + Immutable: true, } if err := scan.EnsureScanner(reg); err != nil { diff --git a/src/pkg/scan/api/scan/base_controller.go b/src/pkg/scan/api/scan/base_controller.go index 96271521f..e33f2b54d 100644 --- a/src/pkg/scan/api/scan/base_controller.go +++ b/src/pkg/scan/api/scan/base_controller.go @@ -378,7 +378,7 @@ func (bc *basicController) makeBasicAuthorization(pid int64, repository string, resource := rbac.NewProjectNamespace(pid).Resource(rbac.ResourceRepository) access := []*rbac.Policy{{ Resource: resource, - Action: rbac.ActionPull, + Action: rbac.ActionScannerPull, }} robotReq := &model.RobotCreate{ @@ -481,7 +481,7 @@ func makeBearerAuthorization(repository string, username string) (string, error) { Type: "repository", Name: repository, - Actions: []string{"pull"}, + Actions: []string{rbac.ActionPull.String(), rbac.ActionScannerPull.String()}, }, } diff --git a/src/pkg/scan/api/scan/base_controller_test.go b/src/pkg/scan/api/scan/base_controller_test.go index c256c4b23..acd901f39 100644 --- a/src/pkg/scan/api/scan/base_controller_test.go +++ b/src/pkg/scan/api/scan/base_controller_test.go @@ -161,7 +161,7 @@ func (suite *ControllerTestSuite) SetupSuite() { resource := fmt.Sprintf("/project/%d/repository", suite.artifact.NamespaceID) access := []*rbac.Policy{{ Resource: rbac.Resource(resource), - Action: "pull", + Action: rbac.ActionScannerPull, }} rname := "the-uuid-123" diff --git a/tests/hostcfg.sh b/tests/hostcfg.sh index ac8bc8966..35ac78131 100755 --- a/tests/hostcfg.sh +++ b/tests/hostcfg.sh @@ -7,7 +7,3 @@ sudo sed "s/reg.mydomain.com/$IP/" -i make/harbor.yml echo "https:" >> make/harbor.yml echo " certificate: /data/cert/server.crt" >> make/harbor.yml echo " private_key: /data/cert/server.key" >> make/harbor.yml - -# TODO: remove it when scanner adapter support internal access of harbor -echo "storage_service:" >> make/harbor.yml -echo " ca_bundle: /data/cert/server.crt" >> make/harbor.yml