Merge pull request #9434 from heww/clair-adapter

build(clair): internal clair adapter when install with clair

Makefile

@@ -106,6 +106,7 @@ CLAIRDBVERSION=$(VERSIONTAG)
 MIGRATORVERSION=$(VERSIONTAG)
 REDISVERSION=$(VERSIONTAG)
 NOTARYMIGRATEVERSION=v3.5.4
+CLAIRADAPTERVERSION=c7db8b15
 
 # version of chartmuseum
 CHARTMUSEUMVERSION=v0.9.0
@@ -115,6 +116,7 @@ VERSION_TAG: $(VERSIONTAG)
 REGISTRY_VERSION: $(REGISTRYVERSION)
 NOTARY_VERSION: $(NOTARYVERSION)
 CLAIR_VERSION: $(CLAIRVERSION)
+CLAIR_ADAPTER_VERSION: $(CLAIRADAPTERVERSION)
 CHARTMUSEUM_VERSION: $(CHARTMUSEUMVERSION)
 endef
 
@@ -251,7 +253,7 @@ ifeq ($(NOTARYFLAG), true)
 	DOCKERSAVE_PARA+= goharbor/notary-server-photon:$(NOTARYVERSION)-$(VERSIONTAG) goharbor/notary-signer-photon:$(NOTARYVERSION)-$(VERSIONTAG)
 endif
 ifeq ($(CLAIRFLAG), true)
-	DOCKERSAVE_PARA+= goharbor/clair-photon:$(CLAIRVERSION)-$(VERSIONTAG)
+	DOCKERSAVE_PARA+= goharbor/clair-photon:$(CLAIRVERSION)-$(VERSIONTAG) goharbor/clair-adapter-photon:$(CLAIRADAPTERVERSION)-$(VERSIONTAG)
 endif
 ifeq ($(MIGRATORFLAG), true)
 	DOCKERSAVE_PARA+= goharbor/harbor-migrator:$(MIGRATORVERSION)
@@ -305,7 +307,7 @@ prepare: update_prepare_version
 build:
 	make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) \
 	 -e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
-	 -e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \
+	 -e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRADAPTERVERSION=$(CLAIRADAPTERVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \
 	 -e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \
 	 -e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER) \
 	 -e NPM_REGISTRY=$(NPM_REGISTRY)

docs/swagger.yaml

@@ -4008,7 +4008,7 @@ paths:
           schema:
             type: array
             items:
-            $ref: '#/definitions/ImmutableTagRule'
+              $ref: '#/definitions/ImmutableTagRule'
         '400':
           description: Illegal format of provided ID value.
         '401':
@@ -5115,35 +5115,7 @@ definitions:
         description: 'The signature of image, defined by RepoSignature. If it is null, the image is unsigned.'
       scan_overview:
         type: object
-        description: The overview of the scan result.  This is an optional property.
+        description: The overview of the scan result.
-        properties:
-          digest:
-            type: string
-            description: The digest of the image.
-          scan_status:
-            type: string
-            description: 'The status of the scan job, it can be "pending", "running", "finished", "error".'
-          job_id:
-            type: integer
-            description: The ID of the job on jobservice to scan the image.
-          severity:
-            type: integer
-            description: '0-Not scanned, 1-Negligible, 2-Unknown, 3-Low, 4-Medium, 5-High'
-          details_key:
-            type: string
-            description: 'The top layer name of this image in Clair, this is for calling Clair API to get the vulnerability list of this image.'
-          components:
-            type: object
-            description: The components overview of the image.
-            properties:
-              total:
-                type: integer
-                description: Total number of the components in this image.
-              summary:
-                description: List of number of components of different severities.
-                type: array
-                items:
-                  $ref: '#/definitions/ComponentOverviewEntry'
       labels:
         type: array
         description: The label list.
@@ -6274,12 +6246,10 @@ definitions:
           $ref: '#/definitions/RetentionRule'
       trigger:
         type: object
-        items:
+        $ref: '#/definitions/RetentionRuleTrigger'
-          $ref: '#/definitions/RetentionRuleTrigger'
       scope:
         type: object
-        items:
+        $ref: '#/definitions/RetentionPolicyScope'
-          $ref: '#/definitions/RetentionPolicyScope'
 
   RetentionRuleTrigger:
     type: object

make/photon/Makefile

@@ -64,6 +64,10 @@ DOCKERFILEPATH_CLAIR=$(DOCKERFILEPATH)/clair
 DOCKERFILENAME_CLAIR=Dockerfile
 DOCKERIMAGENAME_CLAIR=goharbor/clair-photon
 
+DOCKERFILEPATH_CLAIR_ADAPTER=$(DOCKERFILEPATH)/clair-adapter
+DOCKERFILENAME_CLAIR_ADAPTER=Dockerfile
+DOCKERIMAGENAME_CLAIR_ADAPTER=goharbor/clair-adapter-photon
+
 DOCKERFILEPATH_NGINX=$(DOCKERFILEPATH)/nginx
 DOCKERFILENAME_NGINX=Dockerfile
 DOCKERIMAGENAME_NGINX=goharbor/nginx-photon
@@ -141,6 +145,16 @@ _build_clair:
 		echo "Done." ; \
 	fi
 
+_build_clair_adapter:
+	# TODO: add support to fetch clair adapter binary from google storage ranther than build from source
+	@if [ "$(CLAIRFLAG)" = "true" ] ; then \
+		cd $(DOCKERFILEPATH_CLAIR_ADAPTER) && $(DOCKERFILEPATH_CLAIR_ADAPTER)/builder $(CLAIRADAPTERVERSION) && cd - ; \
+		echo "building clair adapter container for photon..." ; \
+		$(DOCKERBUILD) -f $(DOCKERFILEPATH_CLAIR_ADAPTER)/$(DOCKERFILENAME_CLAIR_ADAPTER) -t $(DOCKERIMAGENAME_CLAIR_ADAPTER):$(CLAIRADAPTERVERSION)-$(VERSIONTAG) . ; \
+		rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary; \
+		echo "Done." ; \
+	fi
+
 _build_chart_server:
 	@if [ "$(CHARTFLAG)" = "true" ] ; then \
 		if [ "$(BUILDBIN)" != "true" ] ; then \
@@ -209,7 +223,7 @@ define _get_binary
 	$(WGET) --timeout 30 --no-check-certificate $1 -O $2
 endef
 
-build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_redis _build_migrator _build_chart_server
+build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_clair_adapter _build_redis _build_migrator _build_chart_server
 
 cleanimage:
 	@echo "cleaning image for photon..."

make/photon/clair-adapter/Dockerfile

@@ -0,0 +1,20 @@
+FROM photon:2.0
+
+RUN tdnf install -y sudo >>/dev/null\
+    && tdnf clean all \
+    && mkdir /clair-adapter/ \
+    && groupadd -r -g 10000 clair-adapter \
+    && useradd --no-log-init -m -r -g 10000 -u 10000 clair-adapter
+
+COPY ./make/photon/clair-adapter/binary/harbor-scanner-clair /clair-adapter/clair-adapter
+
+RUN chown -R 10000:10000 /clair-adapter \
+    && chmod u+x /clair-adapter/clair-adapter
+
+EXPOSE 8080
+
+HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS 127.0.0.1:8080/healthy || exit 1
+
+USER clair-adapter
+
+ENTRYPOINT ["/clair-adapter/clair-adapter"]

make/photon/clair-adapter/Dockerfile.binary

@@ -0,0 +1,7 @@
+FROM golang:1.12.5
+
+ADD .   /go/src/github.com/goharbor/harbor-scanner-clair/
+WORKDIR /go/src/github.com/goharbor/harbor-scanner-clair/
+
+RUN export GOOS=linux GO111MODULE=on CGO_ENABLED=0 && \
+	go build github.com/goharbor/harbor-scanner-clair/cmd/harbor-scanner-clair

make/photon/clair-adapter/builder

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set +e
+
+if [ -z $1 ]; then
+  error "Please set the 'version' variable"
+  exit 1
+fi
+
+VERSION="$1"
+
+set -e
+
+# the temp folder to store binary file...
+mkdir -p binary
+rm -rf binary/harbor-scanner-clair || true
+
+cd `dirname $0`
+cur=$PWD
+
+# the temp folder to store distribution source code...
+TEMP=`mktemp -d ${TMPDIR-/tmp}/clair-adapter.XXXXXX`
+git clone https://github.com/danielpacak/harbor-scanner-clair.git $TEMP
+cd $TEMP; git checkout $VERSION; cd -
+
+echo 'build the clair adapter binary bases on the golang:1.12.5...'
+cp Dockerfile.binary $TEMP
+docker build -f $TEMP/Dockerfile.binary -t clair-adapter-golang $TEMP
+
+echo 'copy the clair adapter binary to local...'
+ID=$(docker create clair-adapter-golang)
+docker cp $ID:/go/src/github.com/goharbor/harbor-scanner-clair/harbor-scanner-clair binary
+
+docker rm -f $ID
+docker rmi -f clair-adapter-golang
+
+echo "Build clair adapter binary success, then to build photon image..."
+cd $cur
+rm -rf $TEMP

make/photon/prepare/main.py

@@ -13,6 +13,7 @@ from utils.core import prepare_core
 from utils.notary import prepare_notary
 from utils.log import prepare_log_configs
 from utils.clair import prepare_clair
+from utils.clair_adapter import prepare_clair_adapter
 from utils.chart import prepare_chartmuseum
 from utils.docker_compose import prepare_docker_compose
 from utils.nginx import prepare_nginx, nginx_confd_dir
@@ -54,6 +55,7 @@ def main(conf, with_notary, with_clair, with_chartmuseum):
 
     if with_clair:
         prepare_clair(config_dict)
+        prepare_clair_adapter(config_dict)
 
     if with_chartmuseum:
         prepare_chartmuseum(config_dict)

make/photon/prepare/templates/clair-adapter/env.jinja

@@ -0,0 +1 @@
+SCANNER_CLAIR_URL={{clair_url}}

make/photon/prepare/templates/core/env.jinja

@@ -36,6 +36,7 @@ CORE_URL={{core_url}}
 CORE_LOCAL_URL={{core_local_url}}
 JOBSERVICE_URL={{jobservice_url}}
 CLAIR_URL={{clair_url}}
+CLAIR_ADAPTER_URL={{clair_adapter_url}}
 NOTARY_URL={{notary_url}}
 REGISTRY_STORAGE_PROVIDER_NAME={{storage_provider_name}}
 READ_ONLY=false

make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja

@@ -56,7 +56,7 @@ services:
       - log
     logging:
       driver: "syslog"
-      options:  
+      options:
         syslog-address: "tcp://127.0.0.1:1514"
         tag: "registry"
   registryctl:
@@ -89,7 +89,7 @@ services:
       - log
     logging:
       driver: "syslog"
-      options:  
+      options:
         syslog-address: "tcp://127.0.0.1:1514"
         tag: "registryctl"
 {% if external_database == False %}
@@ -125,7 +125,7 @@ services:
       - log
     logging:
       driver: "syslog"
-      options:  
+      options:
         syslog-address: "tcp://127.0.0.1:1514"
         tag: "postgresql"
 {% endif %}
@@ -186,7 +186,7 @@ services:
 {% endif %}
     logging:
       driver: "syslog"
-      options:  
+      options:
         syslog-address: "tcp://127.0.0.1:1514"
         tag: "core"
   portal:
@@ -307,7 +307,7 @@ services:
       - log
     logging:
       driver: "syslog"
-      options:  
+      options:
         syslog-address: "tcp://127.0.0.1:1514"
         tag: "proxy"
 {% if with_notary %}
@@ -336,7 +336,7 @@ services:
       - notary-signer
     logging:
       driver: "syslog"
-      options:  
+      options:
         syslog-address: "tcp://127.0.0.1:1514"
         tag: "notary-server"
   notary-signer:
@@ -366,7 +366,7 @@ services:
   {% endif %}
     logging:
       driver: "syslog"
-      options:  
+      options:
         syslog-address: "tcp://127.0.0.1:1514"
         tag: "notary-signer"
 {% endif %}
@@ -406,6 +406,29 @@ services:
         tag: "clair"
     env_file:
       ./common/config/clair/clair_env
+  clair-adapter:
+    networks:
+      - harbor-clair
+    container_name: clair-adapter
+    image: goharbor/clair-adapter-photon:{{clair_adapter_version}}
+    restart: always
+    cap_drop:
+      - ALL
+    cap_add:
+      - DAC_OVERRIDE
+      - SETGID
+      - SETUID
+    cpu_quota: 50000
+    dns_search: .
+    depends_on:
+      - clair
+    logging:
+      driver: "syslog"
+      options:
+        syslog-address: "tcp://127.0.0.1:1514"
+        tag: "clair-adapter"
+    env_file:
+      ./common/config/clair-adapter/env
 {% endif %}
 {% if with_chartmuseum %}
   chartmuseum:
@@ -439,7 +462,7 @@ services:
 {% endif %}
     logging:
       driver: "syslog"
-      options:  
+      options:
         syslog-address: "tcp://127.0.0.1:1514"
         tag: "chartmuseum"
     env_file:

make/photon/prepare/utils/clair_adapter.py

@@ -0,0 +1,18 @@
+import os
+
+from g import templates_dir, config_dir
+from .jinja import render_jinja
+from .misc import prepare_dir
+
+clair_adapter_template_dir = os.path.join(templates_dir, "clair-adapter")
+
+def prepare_clair_adapter(config_dict):
+    clair_adapter_config_dir = prepare_dir(config_dir, "clair-adapter")
+
+    clair_adapter_env_path = os.path.join(clair_adapter_config_dir, "env")
+    clair_adapter_env_template = os.path.join(clair_adapter_template_dir, "env.jinja")
+
+    render_jinja(
+        clair_adapter_env_template,
+        clair_adapter_env_path,
+        **config_dict)

make/photon/prepare/utils/configs.py

@@ -74,6 +74,7 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_chartmuseu
         'token_service_url': "http://core:8080/service/token",
         'jobservice_url': 'http://jobservice:8080',
         'clair_url': 'http://clair:6060',
+        'clair_adapter_url': 'http://clair-adapter:8080',
         'notary_url': 'http://notary-server:4443',
         'chart_repository_url': 'http://chartmuseum:9999'
     }

make/photon/prepare/utils/docker_compose.py

@@ -11,9 +11,10 @@ docker_compose_yml_path = '/compose_location/docker-compose.yml'
 def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):
     versions = parse_versions()
     VERSION_TAG = versions.get('VERSION_TAG') or 'dev'
-    REGISTRY_VERSION = versions.get('REGISTRY_VERSION') or 'v2.7.1'
+    REGISTRY_VERSION = versions.get('REGISTRY_VERSION') or 'v2.7.1-patch-2819-2553'
     NOTARY_VERSION = versions.get('NOTARY_VERSION') or 'v0.6.1'
     CLAIR_VERSION = versions.get('CLAIR_VERSION') or 'v2.0.9'
+    CLAIR_ADAPTER_VERSION = versions.get('CLAIR_ADAPTER_VERSION') or ''
     CHARTMUSEUM_VERSION = versions.get('CHARTMUSEUM_VERSION') or 'v0.9.0'
 
     rendering_variables = {
@@ -22,6 +23,7 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):
         'redis_version': VERSION_TAG,
         'notary_version': '{}-{}'.format(NOTARY_VERSION, VERSION_TAG),
         'clair_version': '{}-{}'.format(CLAIR_VERSION, VERSION_TAG),
+        'clair_adapter_version': '{}-{}'.format(CLAIR_ADAPTER_VERSION, VERSION_TAG),
         'chartmuseum_version': '{}-{}'.format(CHARTMUSEUM_VERSION, VERSION_TAG),
         'data_volume': configs['data_volume'],
         'log_location': configs['log_location'],

src/common/config/metadata/metadatalist.go

@@ -73,6 +73,7 @@ var (
 		{Name: common.ClairDBSSLMode, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_DB_SSLMODE", DefaultValue: "disable", ItemType: &StringType{}, Editable: false},
 		{Name: common.ClairDBUsername, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_DB_USERNAME", DefaultValue: "postgres", ItemType: &StringType{}, Editable: false},
 		{Name: common.ClairURL, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_URL", DefaultValue: "http://clair:6060", ItemType: &StringType{}, Editable: false},
+		{Name: common.ClairAdapterURL, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_ADAPTER_URL", DefaultValue: "http://clair-adapter:8080", ItemType: &StringType{}, Editable: false},
 
 		{Name: common.CoreURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "CORE_URL", DefaultValue: "http://core:8080", ItemType: &StringType{}, Editable: false},
 		{Name: common.CoreLocalURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "CORE_LOCAL_URL", DefaultValue: "http://127.0.0.1:8080", ItemType: &StringType{}, Editable: false},

src/common/const.go

@@ -121,6 +121,7 @@ const (
 	GroupMember                       = "g"
 	ReadOnly                          = "read_only"
 	ClairURL                          = "clair_url"
+	ClairAdapterURL                   = "clair_adapter_url"
 	NotaryURL                         = "notary_url"
 	DefaultCoreEndpoint               = "http://core:8080"
 	DefaultNotaryEndpoint             = "http://notary-server:4443"

src/core/config/config.go

@@ -386,6 +386,11 @@ func ClairDB() (*models.PostGreSQL, error) {
 	return clairDB, nil
 }
 
+// ClairAdapterEndpoint returns the endpoint of clair adapter instance, by default it's the one deployed within Harbor.
+func ClairAdapterEndpoint() string {
+	return cfgMgr.Get(common.ClairAdapterURL).GetString()
+}
+
 // AdmiralEndpoint returns the URL of admiral, if Harbor is not deployed with admiral it should return an empty string.
 func AdmiralEndpoint() string {
 	if cfgMgr.Get(common.AdmiralEndpoint).GetString() == "NA" {

src/core/main.go

@@ -48,6 +48,8 @@ import (
 	_ "github.com/goharbor/harbor/src/core/notifier/topic"
 	"github.com/goharbor/harbor/src/core/service/token"
 	"github.com/goharbor/harbor/src/pkg/notification"
+	"github.com/goharbor/harbor/src/pkg/scan"
+	"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
 	"github.com/goharbor/harbor/src/pkg/scheduler"
 	"github.com/goharbor/harbor/src/pkg/types"
 	"github.com/goharbor/harbor/src/replication"
@@ -215,6 +217,19 @@ func main() {
 		if err := dao.InitClairDB(clairDB); err != nil {
 			log.Fatalf("failed to initialize clair database: %v", err)
 		}
+
+		// TODO: change to be internal adapter
+		reg := &scanner.Registration{
+			Name:        "Clair",
+			Description: "The clair scanner adapter",
+			URL:         config.ClairAdapterEndpoint(),
+			Disabled:    false,
+			IsDefault:   true,
+		}
+
+		if err := scan.EnsureScanner(reg); err != nil {
+			log.Fatalf("failed to initialize clair scanner: %v", err)
+		}
 	}
 
 	closing := make(chan struct{})

src/pkg/scan/api/scan/base_controller.go

@@ -15,10 +15,10 @@
 package scan
 
 import (
+	"encoding/base64"
 	"fmt"
 	"time"
 
-	"github.com/goharbor/harbor/src/common"
 	cj "github.com/goharbor/harbor/src/common/job"
 	jm "github.com/goharbor/harbor/src/common/job/models"
 	"github.com/goharbor/harbor/src/common/rbac"
@@ -102,6 +102,14 @@ func NewController() Controller {
 	}
 }
 
+func (bc *basicController) jobClient() cj.Client {
+	if bc.jc == nil {
+		return cj.GlobalClient
+	}
+
+	return bc.jc
+}
+
 // Scan ...
 func (bc *basicController) Scan(artifact *v1.Artifact) error {
 	if artifact == nil {
@@ -276,7 +284,7 @@ func (bc *basicController) GetScanLog(uuid string) ([]byte, error) {
 	}
 
 	// Job log
-	return bc.jc.GetJobLog(sr.JobID)
+	return bc.jobClient().GetJobLog(sr.JobID)
 }
 
 // HandleJobHooks ...
@@ -321,8 +329,8 @@ func (bc *basicController) HandleJobHooks(trackID string, change *job.StatusChan
 	return bc.manager.UpdateStatus(trackID, change.Status, change.Metadata.Revision)
 }
 
-// makeRobotAccount creates a robot account based on the arguments for scanning.
+// makeAuthorization creates authorization from a robot account based on the arguments for scanning.
-func (bc *basicController) makeRobotAccount(pid int64, repository string, ttl int64) (string, error) {
+func (bc *basicController) makeAuthorization(pid int64, repository string, ttl int64) (string, error) {
 	// Use uuid as name to avoid duplicated entries.
 	UUID, err := bc.uuid()
 	if err != nil {
@@ -333,25 +341,28 @@ func (bc *basicController) makeRobotAccount(pid int64, repository string, ttl in
 
 	logger.Warningf("repository %s and expire time %d are not supported by robot controller", repository, expireAt)
 
-	resource := fmt.Sprintf("/project/%d/repository", pid)
+	resource := rbac.NewProjectNamespace(pid).Resource(rbac.ResourceRepository)
 	access := []*rbac.Policy{{
-		Resource: rbac.Resource(resource),
+		Resource: resource,
-		Action:   "pull",
+		Action:   rbac.ActionPull,
 	}}
 
-	account := &model.RobotCreate{
+	robotReq := &model.RobotCreate{
-		Name:        fmt.Sprintf("%s%s", common.RobotPrefix, UUID),
+		Name:        UUID,
 		Description: "for scan",
 		ProjectID:   pid,
 		Access:      access,
 	}
 
-	rb, err := bc.rc.CreateRobotAccount(account)
+	rb, err := bc.rc.CreateRobotAccount(robotReq)
 	if err != nil {
 		return "", errors.Wrap(err, "scan controller: make robot account")
 	}
 
-	return rb.Token, nil
+	username := rb.Name
+	password := rb.Token
+
+	return "Basic " + base64.StdEncoding.EncodeToString([]byte(username+":"+password)), nil
 }
 
 // launchScanJob launches a job to run scan
@@ -361,8 +372,8 @@ func (bc *basicController) launchScanJob(trackID string, artifact *v1.Artifact,
 		return "", errors.Wrap(err, "scan controller: launch scan job")
 	}
 
-	// Make a robot account with 30 minutes
+	// Make authorization from a robot account with 30 minutes
-	robotAccount, err := bc.makeRobotAccount(artifact.NamespaceID, artifact.Repository, 1800)
+	authorization, err := bc.makeAuthorization(artifact.NamespaceID, artifact.Repository, 1800)
 	if err != nil {
 		return "", errors.Wrap(err, "scan controller: launch scan job")
 	}
@@ -371,7 +382,7 @@ func (bc *basicController) launchScanJob(trackID string, artifact *v1.Artifact,
 	scanReq := &v1.ScanRequest{
 		Registry: &v1.Registry{
 			URL:           externalURL,
-			Authorization: robotAccount,
+			Authorization: authorization,
 		},
 		Artifact: artifact,
 	}
@@ -407,5 +418,5 @@ func (bc *basicController) launchScanJob(trackID string, artifact *v1.Artifact,
 		StatusHook: hookURL,
 	}
 
-	return bc.jc.SubmitJob(j)
+	return bc.jobClient().SubmitJob(j)
 }

src/pkg/scan/api/scan/base_controller_test.go

@@ -15,6 +15,7 @@
 package scan
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"testing"
@@ -164,7 +165,7 @@ func (suite *ControllerTestSuite) SetupSuite() {
 		Action:   "pull",
 	}}
 
-	rname := fmt.Sprintf("%s%s", common.RobotPrefix, "the-uuid-123")
+	rname := "the-uuid-123"
 	account := &model.RobotCreate{
 		Name:        rname,
 		Description: "for scan",
@@ -173,7 +174,7 @@ func (suite *ControllerTestSuite) SetupSuite() {
 	}
 	rc.On("CreateRobotAccount", account).Return(&model.Robot{
 		ID:          1,
-		Name:        rname,
+		Name:        common.RobotPrefix + rname,
 		Token:       "robot-account",
 		Description: "for scan",
 		ProjectID:   suite.artifact.NamespaceID,
@@ -183,7 +184,7 @@ func (suite *ControllerTestSuite) SetupSuite() {
 	req := &v1.ScanRequest{
 		Registry: &v1.Registry{
 			URL:           "https://core.com",
-			Authorization: "robot-account",
+			Authorization: "Basic " + base64.StdEncoding.EncodeToString([]byte(common.RobotPrefix+"the-uuid-123:robot-account")),
 		},
 		Artifact: suite.artifact,
 	}

src/pkg/scan/init.go

@@ -0,0 +1,44 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scan
+
+import (
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/pkg/q"
+	sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner"
+	"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
+)
+
+// EnsureScanner ensure the scanner which specially name exists in the system
+func EnsureScanner(registration *scanner.Registration) error {
+	q := &q.Query{
+		Keywords: map[string]interface{}{"url": registration.URL},
+	}
+
+	registrations, err := sc.DefaultController.ListRegistrations(q)
+	if err != nil {
+		return err
+	}
+
+	if len(registrations) == 0 {
+		if _, err := sc.DefaultController.CreateRegistration(registration); err != nil {
+			return err
+		}
+
+		log.Infof("initialized scanner named %s", registration.Name)
+	}
+
+	return nil
+}

tests/apitests/python/library/base.py

@@ -3,6 +3,10 @@
 import sys
 import time
 import swagger_client
+try:
+    from urllib import getproxies
+except ImportError:
+    from urllib.request import getproxies
 
 class Server:
     def __init__(self, endpoint, verify_ssl):
@@ -23,6 +27,12 @@ def _create_client(server, credential, debug):
     cfg.username = credential.username
     cfg.password = credential.password
     cfg.debug = debug
+
+    proxies = getproxies()
+    proxy = proxies.get('http', proxies.get('all', None))
+    if proxy:
+        cfg.proxy = proxy
+
     return swagger_client.ProductsApi(swagger_client.ApiClient(cfg))
 
 def _assert_status_code(expect_code, return_code):

tests/apitests/python/library/repository.py

@@ -100,7 +100,7 @@ class Repository(base.Base):
         if tag.scan_overview != None:
             raise Exception("Image should be <Not Scanned> state!")
 
-    def check_image_scan_result(self, repo_name, tag, expected_scan_status = "finished", **kwargs):
+    def check_image_scan_result(self, repo_name, tag, expected_scan_status = "Success", **kwargs):
         timeout_count = 30
         while True:
             time.sleep(5)
@@ -108,12 +108,13 @@ class Repository(base.Base):
             if (timeout_count == 0):
                 break
             _tag = self.get_tag(repo_name, tag, **kwargs)
-            if _tag.name == tag and _tag.scan_overview !=None:
+            if _tag.name == tag and _tag.scan_overview != None:
-                if _tag.scan_overview.scan_status == expected_scan_status:
+                for report in _tag.scan_overview.values():
-                    return
+                    if report.get('scan_status') == expected_scan_status:
+                        return
         raise Exception("Scan image result is not as expected {}.".format(expected_scan_status))
 
-    def scan_image(self, repo_name, tag, expect_status_code = 200, **kwargs):
+    def scan_image(self, repo_name, tag, expect_status_code = 202, **kwargs):
         client = self._get_client(**kwargs)
         data, status_code, _ = client.repositories_repo_name_tags_tag_scan_post_with_http_info(repo_name, tag)
         base._assert_status_code(expect_status_code, status_code)

tests/apitests/python/test_scan_all_images.py

@@ -93,8 +93,8 @@ class TestProjects(unittest.TestCase):
         self.system.scan_now(**ADMIN_CLIENT)
 
         #5. Check if image in project_Alice and another image in project_Luca were both scanned.
-        self.repo.check_image_scan_result(TestProjects.repo_Alice_name, tag_Alice, expected_scan_status = "finished", **USER_ALICE_CLIENT)
+        self.repo.check_image_scan_result(TestProjects.repo_Alice_name, tag_Alice, **USER_ALICE_CLIENT)
-        self.repo.check_image_scan_result(TestProjects.repo_Luca_name, tag_Luca, expected_scan_status = "finished", **USER_LUCA_CLIENT)
+        self.repo.check_image_scan_result(TestProjects.repo_Luca_name, tag_Luca, **USER_LUCA_CLIENT)
 
 if __name__ == '__main__':
     unittest.main()

tests/apitests/python/test_scan_image.py

@@ -80,7 +80,7 @@ class TestProjects(unittest.TestCase):
 
         #6. Send scan image command and get tag(TA) information to check scan result, it should be finished;
         self.repo.scan_image(TestProjects.repo_name, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT)
-        self.repo.check_image_scan_result(TestProjects.repo_name, tag, expected_scan_status = "finished", **TestProjects.USER_SCAN_IMAGE_CLIENT)
+        self.repo.check_image_scan_result(TestProjects.repo_name, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT)
 
 if __name__ == '__main__':
     unittest.main()

tests/hostcfg.sh

@@ -6,4 +6,8 @@ sudo sed "s/reg.mydomain.com/$IP/" -i make/harbor.yml
 
 echo "https:" >> make/harbor.yml
 echo "  certificate: /data/cert/server.crt" >> make/harbor.yml
-echo "  private_key: /data/cert/server.key" >> make/harbor.yml
+echo "  private_key: /data/cert/server.key" >> make/harbor.yml
+
+# TODO: remove it when scanner adapter support internal access of harbor
+echo "storage_service:" >> make/harbor.yml
+echo "  ca_bundle: /data/cert/server.crt" >> make/harbor.yml

tests/robot-cases/Group0-BAT/API_DB.robot

@@ -29,18 +29,16 @@ Test Case - Add Replication Rule
     Harbor API Test  ./tests/apitests/python/test_add_replication_rule.py
 Test Case - Edit Project Creation
     Harbor API Test  ./tests/apitests/python/test_edit_project_creation.py
-*** Enable this case after deployment change PR merged ***
+Test Case - Scan Image
-*** Test Case - Scan Image ***
+    Harbor API Test  ./tests/apitests/python/test_scan_image.py
-***    Harbor API Test  ./tests/apitests/python/test_scan_image.py ***
 Test Case - Manage Project Member
     Harbor API Test  ./tests/apitests/python/test_manage_project_member.py
 Test Case - Project Level Policy Content Trust
     Harbor API Test  ./tests/apitests/python/test_project_level_policy_content_trust.py
 Test Case - User View Logs
     Harbor API Test  ./tests/apitests/python/test_user_view_logs.py
-*** Enable this case after deployment change PR merged ***
+Test Case - Scan All Images
-*** Test Case - Scan All Images ***
+    Harbor API Test  ./tests/apitests/python/test_scan_all_images.py
-***    Harbor API Test  ./tests/apitests/python/test_scan_all_images.py ***
 Test Case - List Helm Charts
     Harbor API Test  ./tests/apitests/python/test_list_helm_charts.py
 Test Case - Assign Sys Admin