diff --git a/Makefile b/Makefile index c18b29cf7..4e8b9bcac 100644 --- a/Makefile +++ b/Makefile @@ -106,6 +106,7 @@ CLAIRDBVERSION=$(VERSIONTAG) MIGRATORVERSION=$(VERSIONTAG) REDISVERSION=$(VERSIONTAG) NOTARYMIGRATEVERSION=v3.5.4 +CLAIRADAPTERVERSION=c7db8b15 # version of chartmuseum CHARTMUSEUMVERSION=v0.9.0 @@ -115,6 +116,7 @@ VERSION_TAG: $(VERSIONTAG) REGISTRY_VERSION: $(REGISTRYVERSION) NOTARY_VERSION: $(NOTARYVERSION) CLAIR_VERSION: $(CLAIRVERSION) +CLAIR_ADAPTER_VERSION: $(CLAIRADAPTERVERSION) CHARTMUSEUM_VERSION: $(CHARTMUSEUMVERSION) endef @@ -251,7 +253,7 @@ ifeq ($(NOTARYFLAG), true) DOCKERSAVE_PARA+= goharbor/notary-server-photon:$(NOTARYVERSION)-$(VERSIONTAG) goharbor/notary-signer-photon:$(NOTARYVERSION)-$(VERSIONTAG) endif ifeq ($(CLAIRFLAG), true) - DOCKERSAVE_PARA+= goharbor/clair-photon:$(CLAIRVERSION)-$(VERSIONTAG) + DOCKERSAVE_PARA+= goharbor/clair-photon:$(CLAIRVERSION)-$(VERSIONTAG) goharbor/clair-adapter-photon:$(CLAIRADAPTERVERSION)-$(VERSIONTAG) endif ifeq ($(MIGRATORFLAG), true) DOCKERSAVE_PARA+= goharbor/harbor-migrator:$(MIGRATORVERSION) @@ -305,7 +307,7 @@ prepare: update_prepare_version build: make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) \ -e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \ - -e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \ + -e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRADAPTERVERSION=$(CLAIRADAPTERVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \ -e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \ -e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER) \ -e NPM_REGISTRY=$(NPM_REGISTRY) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index e2a83c8ab..6b85f001b 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -4008,7 +4008,7 @@ paths: schema: type: array items: - $ref: '#/definitions/ImmutableTagRule' + $ref: '#/definitions/ImmutableTagRule' '400': description: Illegal format of provided ID value. '401': @@ -5115,35 +5115,7 @@ definitions: description: 'The signature of image, defined by RepoSignature. If it is null, the image is unsigned.' scan_overview: type: object - description: The overview of the scan result. This is an optional property. - properties: - digest: - type: string - description: The digest of the image. - scan_status: - type: string - description: 'The status of the scan job, it can be "pending", "running", "finished", "error".' - job_id: - type: integer - description: The ID of the job on jobservice to scan the image. - severity: - type: integer - description: '0-Not scanned, 1-Negligible, 2-Unknown, 3-Low, 4-Medium, 5-High' - details_key: - type: string - description: 'The top layer name of this image in Clair, this is for calling Clair API to get the vulnerability list of this image.' - components: - type: object - description: The components overview of the image. - properties: - total: - type: integer - description: Total number of the components in this image. - summary: - description: List of number of components of different severities. - type: array - items: - $ref: '#/definitions/ComponentOverviewEntry' + description: The overview of the scan result. labels: type: array description: The label list. @@ -6274,12 +6246,10 @@ definitions: $ref: '#/definitions/RetentionRule' trigger: type: object - items: - $ref: '#/definitions/RetentionRuleTrigger' + $ref: '#/definitions/RetentionRuleTrigger' scope: type: object - items: - $ref: '#/definitions/RetentionPolicyScope' + $ref: '#/definitions/RetentionPolicyScope' RetentionRuleTrigger: type: object diff --git a/make/photon/Makefile b/make/photon/Makefile index 76dde92de..a1290cb1e 100644 --- a/make/photon/Makefile +++ b/make/photon/Makefile @@ -64,6 +64,10 @@ DOCKERFILEPATH_CLAIR=$(DOCKERFILEPATH)/clair DOCKERFILENAME_CLAIR=Dockerfile DOCKERIMAGENAME_CLAIR=goharbor/clair-photon +DOCKERFILEPATH_CLAIR_ADAPTER=$(DOCKERFILEPATH)/clair-adapter +DOCKERFILENAME_CLAIR_ADAPTER=Dockerfile +DOCKERIMAGENAME_CLAIR_ADAPTER=goharbor/clair-adapter-photon + DOCKERFILEPATH_NGINX=$(DOCKERFILEPATH)/nginx DOCKERFILENAME_NGINX=Dockerfile DOCKERIMAGENAME_NGINX=goharbor/nginx-photon @@ -141,6 +145,16 @@ _build_clair: echo "Done." ; \ fi +_build_clair_adapter: + # TODO: add support to fetch clair adapter binary from google storage ranther than build from source + @if [ "$(CLAIRFLAG)" = "true" ] ; then \ + cd $(DOCKERFILEPATH_CLAIR_ADAPTER) && $(DOCKERFILEPATH_CLAIR_ADAPTER)/builder $(CLAIRADAPTERVERSION) && cd - ; \ + echo "building clair adapter container for photon..." ; \ + $(DOCKERBUILD) -f $(DOCKERFILEPATH_CLAIR_ADAPTER)/$(DOCKERFILENAME_CLAIR_ADAPTER) -t $(DOCKERIMAGENAME_CLAIR_ADAPTER):$(CLAIRADAPTERVERSION)-$(VERSIONTAG) . ; \ + rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary; \ + echo "Done." ; \ + fi + _build_chart_server: @if [ "$(CHARTFLAG)" = "true" ] ; then \ if [ "$(BUILDBIN)" != "true" ] ; then \ @@ -209,7 +223,7 @@ define _get_binary $(WGET) --timeout 30 --no-check-certificate $1 -O $2 endef -build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_redis _build_migrator _build_chart_server +build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_clair_adapter _build_redis _build_migrator _build_chart_server cleanimage: @echo "cleaning image for photon..." diff --git a/make/photon/clair-adapter/Dockerfile b/make/photon/clair-adapter/Dockerfile new file mode 100644 index 000000000..1b21ec70d --- /dev/null +++ b/make/photon/clair-adapter/Dockerfile @@ -0,0 +1,20 @@ +FROM photon:2.0 + +RUN tdnf install -y sudo >>/dev/null\ + && tdnf clean all \ + && mkdir /clair-adapter/ \ + && groupadd -r -g 10000 clair-adapter \ + && useradd --no-log-init -m -r -g 10000 -u 10000 clair-adapter + +COPY ./make/photon/clair-adapter/binary/harbor-scanner-clair /clair-adapter/clair-adapter + +RUN chown -R 10000:10000 /clair-adapter \ + && chmod u+x /clair-adapter/clair-adapter + +EXPOSE 8080 + +HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS 127.0.0.1:8080/healthy || exit 1 + +USER clair-adapter + +ENTRYPOINT ["/clair-adapter/clair-adapter"] \ No newline at end of file diff --git a/make/photon/clair-adapter/Dockerfile.binary b/make/photon/clair-adapter/Dockerfile.binary new file mode 100644 index 000000000..87707a3b5 --- /dev/null +++ b/make/photon/clair-adapter/Dockerfile.binary @@ -0,0 +1,7 @@ +FROM golang:1.12.5 + +ADD . /go/src/github.com/goharbor/harbor-scanner-clair/ +WORKDIR /go/src/github.com/goharbor/harbor-scanner-clair/ + +RUN export GOOS=linux GO111MODULE=on CGO_ENABLED=0 && \ + go build github.com/goharbor/harbor-scanner-clair/cmd/harbor-scanner-clair diff --git a/make/photon/clair-adapter/builder b/make/photon/clair-adapter/builder new file mode 100755 index 000000000..857fef0e3 --- /dev/null +++ b/make/photon/clair-adapter/builder @@ -0,0 +1,39 @@ +#!/bin/bash + +set +e + +if [ -z $1 ]; then + error "Please set the 'version' variable" + exit 1 +fi + +VERSION="$1" + +set -e + +# the temp folder to store binary file... +mkdir -p binary +rm -rf binary/harbor-scanner-clair || true + +cd `dirname $0` +cur=$PWD + +# the temp folder to store distribution source code... +TEMP=`mktemp -d ${TMPDIR-/tmp}/clair-adapter.XXXXXX` +git clone https://github.com/danielpacak/harbor-scanner-clair.git $TEMP +cd $TEMP; git checkout $VERSION; cd - + +echo 'build the clair adapter binary bases on the golang:1.12.5...' +cp Dockerfile.binary $TEMP +docker build -f $TEMP/Dockerfile.binary -t clair-adapter-golang $TEMP + +echo 'copy the clair adapter binary to local...' +ID=$(docker create clair-adapter-golang) +docker cp $ID:/go/src/github.com/goharbor/harbor-scanner-clair/harbor-scanner-clair binary + +docker rm -f $ID +docker rmi -f clair-adapter-golang + +echo "Build clair adapter binary success, then to build photon image..." +cd $cur +rm -rf $TEMP diff --git a/make/photon/prepare/main.py b/make/photon/prepare/main.py index bec165455..f717aff2b 100644 --- a/make/photon/prepare/main.py +++ b/make/photon/prepare/main.py @@ -13,6 +13,7 @@ from utils.core import prepare_core from utils.notary import prepare_notary from utils.log import prepare_log_configs from utils.clair import prepare_clair +from utils.clair_adapter import prepare_clair_adapter from utils.chart import prepare_chartmuseum from utils.docker_compose import prepare_docker_compose from utils.nginx import prepare_nginx, nginx_confd_dir @@ -54,6 +55,7 @@ def main(conf, with_notary, with_clair, with_chartmuseum): if with_clair: prepare_clair(config_dict) + prepare_clair_adapter(config_dict) if with_chartmuseum: prepare_chartmuseum(config_dict) diff --git a/make/photon/prepare/templates/clair-adapter/env.jinja b/make/photon/prepare/templates/clair-adapter/env.jinja new file mode 100644 index 000000000..65312c89b --- /dev/null +++ b/make/photon/prepare/templates/clair-adapter/env.jinja @@ -0,0 +1 @@ +SCANNER_CLAIR_URL={{clair_url}} \ No newline at end of file diff --git a/make/photon/prepare/templates/core/env.jinja b/make/photon/prepare/templates/core/env.jinja index d6413678e..0c2393605 100644 --- a/make/photon/prepare/templates/core/env.jinja +++ b/make/photon/prepare/templates/core/env.jinja @@ -36,6 +36,7 @@ CORE_URL={{core_url}} CORE_LOCAL_URL={{core_local_url}} JOBSERVICE_URL={{jobservice_url}} CLAIR_URL={{clair_url}} +CLAIR_ADAPTER_URL={{clair_adapter_url}} NOTARY_URL={{notary_url}} REGISTRY_STORAGE_PROVIDER_NAME={{storage_provider_name}} READ_ONLY=false diff --git a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja index d2263011b..359a1550f 100644 --- a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja +++ b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja @@ -56,7 +56,7 @@ services: - log logging: driver: "syslog" - options: + options: syslog-address: "tcp://127.0.0.1:1514" tag: "registry" registryctl: @@ -89,7 +89,7 @@ services: - log logging: driver: "syslog" - options: + options: syslog-address: "tcp://127.0.0.1:1514" tag: "registryctl" {% if external_database == False %} @@ -125,7 +125,7 @@ services: - log logging: driver: "syslog" - options: + options: syslog-address: "tcp://127.0.0.1:1514" tag: "postgresql" {% endif %} @@ -186,7 +186,7 @@ services: {% endif %} logging: driver: "syslog" - options: + options: syslog-address: "tcp://127.0.0.1:1514" tag: "core" portal: @@ -307,7 +307,7 @@ services: - log logging: driver: "syslog" - options: + options: syslog-address: "tcp://127.0.0.1:1514" tag: "proxy" {% if with_notary %} @@ -336,7 +336,7 @@ services: - notary-signer logging: driver: "syslog" - options: + options: syslog-address: "tcp://127.0.0.1:1514" tag: "notary-server" notary-signer: @@ -366,7 +366,7 @@ services: {% endif %} logging: driver: "syslog" - options: + options: syslog-address: "tcp://127.0.0.1:1514" tag: "notary-signer" {% endif %} @@ -406,6 +406,29 @@ services: tag: "clair" env_file: ./common/config/clair/clair_env + clair-adapter: + networks: + - harbor-clair + container_name: clair-adapter + image: goharbor/clair-adapter-photon:{{clair_adapter_version}} + restart: always + cap_drop: + - ALL + cap_add: + - DAC_OVERRIDE + - SETGID + - SETUID + cpu_quota: 50000 + dns_search: . + depends_on: + - clair + logging: + driver: "syslog" + options: + syslog-address: "tcp://127.0.0.1:1514" + tag: "clair-adapter" + env_file: + ./common/config/clair-adapter/env {% endif %} {% if with_chartmuseum %} chartmuseum: @@ -439,7 +462,7 @@ services: {% endif %} logging: driver: "syslog" - options: + options: syslog-address: "tcp://127.0.0.1:1514" tag: "chartmuseum" env_file: diff --git a/make/photon/prepare/utils/clair_adapter.py b/make/photon/prepare/utils/clair_adapter.py new file mode 100644 index 000000000..8a55d0900 --- /dev/null +++ b/make/photon/prepare/utils/clair_adapter.py @@ -0,0 +1,18 @@ +import os + +from g import templates_dir, config_dir +from .jinja import render_jinja +from .misc import prepare_dir + +clair_adapter_template_dir = os.path.join(templates_dir, "clair-adapter") + +def prepare_clair_adapter(config_dict): + clair_adapter_config_dir = prepare_dir(config_dir, "clair-adapter") + + clair_adapter_env_path = os.path.join(clair_adapter_config_dir, "env") + clair_adapter_env_template = os.path.join(clair_adapter_template_dir, "env.jinja") + + render_jinja( + clair_adapter_env_template, + clair_adapter_env_path, + **config_dict) diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index f43dcbe87..26e4c3de7 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -74,6 +74,7 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_chartmuseu 'token_service_url': "http://core:8080/service/token", 'jobservice_url': 'http://jobservice:8080', 'clair_url': 'http://clair:6060', + 'clair_adapter_url': 'http://clair-adapter:8080', 'notary_url': 'http://notary-server:4443', 'chart_repository_url': 'http://chartmuseum:9999' } diff --git a/make/photon/prepare/utils/docker_compose.py b/make/photon/prepare/utils/docker_compose.py index e4bfcd55b..05a8f507f 100644 --- a/make/photon/prepare/utils/docker_compose.py +++ b/make/photon/prepare/utils/docker_compose.py @@ -11,9 +11,10 @@ docker_compose_yml_path = '/compose_location/docker-compose.yml' def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum): versions = parse_versions() VERSION_TAG = versions.get('VERSION_TAG') or 'dev' - REGISTRY_VERSION = versions.get('REGISTRY_VERSION') or 'v2.7.1' + REGISTRY_VERSION = versions.get('REGISTRY_VERSION') or 'v2.7.1-patch-2819-2553' NOTARY_VERSION = versions.get('NOTARY_VERSION') or 'v0.6.1' CLAIR_VERSION = versions.get('CLAIR_VERSION') or 'v2.0.9' + CLAIR_ADAPTER_VERSION = versions.get('CLAIR_ADAPTER_VERSION') or '' CHARTMUSEUM_VERSION = versions.get('CHARTMUSEUM_VERSION') or 'v0.9.0' rendering_variables = { @@ -22,6 +23,7 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum): 'redis_version': VERSION_TAG, 'notary_version': '{}-{}'.format(NOTARY_VERSION, VERSION_TAG), 'clair_version': '{}-{}'.format(CLAIR_VERSION, VERSION_TAG), + 'clair_adapter_version': '{}-{}'.format(CLAIR_ADAPTER_VERSION, VERSION_TAG), 'chartmuseum_version': '{}-{}'.format(CHARTMUSEUM_VERSION, VERSION_TAG), 'data_volume': configs['data_volume'], 'log_location': configs['log_location'], diff --git a/src/common/config/metadata/metadatalist.go b/src/common/config/metadata/metadatalist.go index bf0b70872..387536975 100644 --- a/src/common/config/metadata/metadatalist.go +++ b/src/common/config/metadata/metadatalist.go @@ -73,6 +73,7 @@ var ( {Name: common.ClairDBSSLMode, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_DB_SSLMODE", DefaultValue: "disable", ItemType: &StringType{}, Editable: false}, {Name: common.ClairDBUsername, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_DB_USERNAME", DefaultValue: "postgres", ItemType: &StringType{}, Editable: false}, {Name: common.ClairURL, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_URL", DefaultValue: "http://clair:6060", ItemType: &StringType{}, Editable: false}, + {Name: common.ClairAdapterURL, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_ADAPTER_URL", DefaultValue: "http://clair-adapter:8080", ItemType: &StringType{}, Editable: false}, {Name: common.CoreURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "CORE_URL", DefaultValue: "http://core:8080", ItemType: &StringType{}, Editable: false}, {Name: common.CoreLocalURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "CORE_LOCAL_URL", DefaultValue: "http://127.0.0.1:8080", ItemType: &StringType{}, Editable: false}, diff --git a/src/common/const.go b/src/common/const.go index 45c2e2692..c051762ff 100755 --- a/src/common/const.go +++ b/src/common/const.go @@ -121,6 +121,7 @@ const ( GroupMember = "g" ReadOnly = "read_only" ClairURL = "clair_url" + ClairAdapterURL = "clair_adapter_url" NotaryURL = "notary_url" DefaultCoreEndpoint = "http://core:8080" DefaultNotaryEndpoint = "http://notary-server:4443" diff --git a/src/core/config/config.go b/src/core/config/config.go index f7dea7f8c..be4050281 100755 --- a/src/core/config/config.go +++ b/src/core/config/config.go @@ -386,6 +386,11 @@ func ClairDB() (*models.PostGreSQL, error) { return clairDB, nil } +// ClairAdapterEndpoint returns the endpoint of clair adapter instance, by default it's the one deployed within Harbor. +func ClairAdapterEndpoint() string { + return cfgMgr.Get(common.ClairAdapterURL).GetString() +} + // AdmiralEndpoint returns the URL of admiral, if Harbor is not deployed with admiral it should return an empty string. func AdmiralEndpoint() string { if cfgMgr.Get(common.AdmiralEndpoint).GetString() == "NA" { diff --git a/src/core/main.go b/src/core/main.go index bf485275f..05145e3c9 100755 --- a/src/core/main.go +++ b/src/core/main.go @@ -48,6 +48,8 @@ import ( _ "github.com/goharbor/harbor/src/core/notifier/topic" "github.com/goharbor/harbor/src/core/service/token" "github.com/goharbor/harbor/src/pkg/notification" + "github.com/goharbor/harbor/src/pkg/scan" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" "github.com/goharbor/harbor/src/pkg/scheduler" "github.com/goharbor/harbor/src/pkg/types" "github.com/goharbor/harbor/src/replication" @@ -215,6 +217,19 @@ func main() { if err := dao.InitClairDB(clairDB); err != nil { log.Fatalf("failed to initialize clair database: %v", err) } + + // TODO: change to be internal adapter + reg := &scanner.Registration{ + Name: "Clair", + Description: "The clair scanner adapter", + URL: config.ClairAdapterEndpoint(), + Disabled: false, + IsDefault: true, + } + + if err := scan.EnsureScanner(reg); err != nil { + log.Fatalf("failed to initialize clair scanner: %v", err) + } } closing := make(chan struct{}) diff --git a/src/pkg/scan/api/scan/base_controller.go b/src/pkg/scan/api/scan/base_controller.go index 514eb9ba5..aa6637ffb 100644 --- a/src/pkg/scan/api/scan/base_controller.go +++ b/src/pkg/scan/api/scan/base_controller.go @@ -15,10 +15,10 @@ package scan import ( + "encoding/base64" "fmt" "time" - "github.com/goharbor/harbor/src/common" cj "github.com/goharbor/harbor/src/common/job" jm "github.com/goharbor/harbor/src/common/job/models" "github.com/goharbor/harbor/src/common/rbac" @@ -102,6 +102,14 @@ func NewController() Controller { } } +func (bc *basicController) jobClient() cj.Client { + if bc.jc == nil { + return cj.GlobalClient + } + + return bc.jc +} + // Scan ... func (bc *basicController) Scan(artifact *v1.Artifact) error { if artifact == nil { @@ -276,7 +284,7 @@ func (bc *basicController) GetScanLog(uuid string) ([]byte, error) { } // Job log - return bc.jc.GetJobLog(sr.JobID) + return bc.jobClient().GetJobLog(sr.JobID) } // HandleJobHooks ... @@ -321,8 +329,8 @@ func (bc *basicController) HandleJobHooks(trackID string, change *job.StatusChan return bc.manager.UpdateStatus(trackID, change.Status, change.Metadata.Revision) } -// makeRobotAccount creates a robot account based on the arguments for scanning. -func (bc *basicController) makeRobotAccount(pid int64, repository string, ttl int64) (string, error) { +// makeAuthorization creates authorization from a robot account based on the arguments for scanning. +func (bc *basicController) makeAuthorization(pid int64, repository string, ttl int64) (string, error) { // Use uuid as name to avoid duplicated entries. UUID, err := bc.uuid() if err != nil { @@ -333,25 +341,28 @@ func (bc *basicController) makeRobotAccount(pid int64, repository string, ttl in logger.Warningf("repository %s and expire time %d are not supported by robot controller", repository, expireAt) - resource := fmt.Sprintf("/project/%d/repository", pid) + resource := rbac.NewProjectNamespace(pid).Resource(rbac.ResourceRepository) access := []*rbac.Policy{{ - Resource: rbac.Resource(resource), - Action: "pull", + Resource: resource, + Action: rbac.ActionPull, }} - account := &model.RobotCreate{ - Name: fmt.Sprintf("%s%s", common.RobotPrefix, UUID), + robotReq := &model.RobotCreate{ + Name: UUID, Description: "for scan", ProjectID: pid, Access: access, } - rb, err := bc.rc.CreateRobotAccount(account) + rb, err := bc.rc.CreateRobotAccount(robotReq) if err != nil { return "", errors.Wrap(err, "scan controller: make robot account") } - return rb.Token, nil + username := rb.Name + password := rb.Token + + return "Basic " + base64.StdEncoding.EncodeToString([]byte(username+":"+password)), nil } // launchScanJob launches a job to run scan @@ -361,8 +372,8 @@ func (bc *basicController) launchScanJob(trackID string, artifact *v1.Artifact, return "", errors.Wrap(err, "scan controller: launch scan job") } - // Make a robot account with 30 minutes - robotAccount, err := bc.makeRobotAccount(artifact.NamespaceID, artifact.Repository, 1800) + // Make authorization from a robot account with 30 minutes + authorization, err := bc.makeAuthorization(artifact.NamespaceID, artifact.Repository, 1800) if err != nil { return "", errors.Wrap(err, "scan controller: launch scan job") } @@ -371,7 +382,7 @@ func (bc *basicController) launchScanJob(trackID string, artifact *v1.Artifact, scanReq := &v1.ScanRequest{ Registry: &v1.Registry{ URL: externalURL, - Authorization: robotAccount, + Authorization: authorization, }, Artifact: artifact, } @@ -407,5 +418,5 @@ func (bc *basicController) launchScanJob(trackID string, artifact *v1.Artifact, StatusHook: hookURL, } - return bc.jc.SubmitJob(j) + return bc.jobClient().SubmitJob(j) } diff --git a/src/pkg/scan/api/scan/base_controller_test.go b/src/pkg/scan/api/scan/base_controller_test.go index 5d3c87d35..6d5156d21 100644 --- a/src/pkg/scan/api/scan/base_controller_test.go +++ b/src/pkg/scan/api/scan/base_controller_test.go @@ -15,6 +15,7 @@ package scan import ( + "encoding/base64" "encoding/json" "fmt" "testing" @@ -164,7 +165,7 @@ func (suite *ControllerTestSuite) SetupSuite() { Action: "pull", }} - rname := fmt.Sprintf("%s%s", common.RobotPrefix, "the-uuid-123") + rname := "the-uuid-123" account := &model.RobotCreate{ Name: rname, Description: "for scan", @@ -173,7 +174,7 @@ func (suite *ControllerTestSuite) SetupSuite() { } rc.On("CreateRobotAccount", account).Return(&model.Robot{ ID: 1, - Name: rname, + Name: common.RobotPrefix + rname, Token: "robot-account", Description: "for scan", ProjectID: suite.artifact.NamespaceID, @@ -183,7 +184,7 @@ func (suite *ControllerTestSuite) SetupSuite() { req := &v1.ScanRequest{ Registry: &v1.Registry{ URL: "https://core.com", - Authorization: "robot-account", + Authorization: "Basic " + base64.StdEncoding.EncodeToString([]byte(common.RobotPrefix+"the-uuid-123:robot-account")), }, Artifact: suite.artifact, } diff --git a/src/pkg/scan/init.go b/src/pkg/scan/init.go new file mode 100644 index 000000000..cbc14c795 --- /dev/null +++ b/src/pkg/scan/init.go @@ -0,0 +1,44 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import ( + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/q" + sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" +) + +// EnsureScanner ensure the scanner which specially name exists in the system +func EnsureScanner(registration *scanner.Registration) error { + q := &q.Query{ + Keywords: map[string]interface{}{"url": registration.URL}, + } + + registrations, err := sc.DefaultController.ListRegistrations(q) + if err != nil { + return err + } + + if len(registrations) == 0 { + if _, err := sc.DefaultController.CreateRegistration(registration); err != nil { + return err + } + + log.Infof("initialized scanner named %s", registration.Name) + } + + return nil +} diff --git a/tests/apitests/python/library/base.py b/tests/apitests/python/library/base.py index 9083e82ba..b1fec5b32 100644 --- a/tests/apitests/python/library/base.py +++ b/tests/apitests/python/library/base.py @@ -3,6 +3,10 @@ import sys import time import swagger_client +try: + from urllib import getproxies +except ImportError: + from urllib.request import getproxies class Server: def __init__(self, endpoint, verify_ssl): @@ -23,6 +27,12 @@ def _create_client(server, credential, debug): cfg.username = credential.username cfg.password = credential.password cfg.debug = debug + + proxies = getproxies() + proxy = proxies.get('http', proxies.get('all', None)) + if proxy: + cfg.proxy = proxy + return swagger_client.ProductsApi(swagger_client.ApiClient(cfg)) def _assert_status_code(expect_code, return_code): diff --git a/tests/apitests/python/library/repository.py b/tests/apitests/python/library/repository.py index ddef01b4b..aae3ee2ac 100644 --- a/tests/apitests/python/library/repository.py +++ b/tests/apitests/python/library/repository.py @@ -100,7 +100,7 @@ class Repository(base.Base): if tag.scan_overview != None: raise Exception("Image should be state!") - def check_image_scan_result(self, repo_name, tag, expected_scan_status = "finished", **kwargs): + def check_image_scan_result(self, repo_name, tag, expected_scan_status = "Success", **kwargs): timeout_count = 30 while True: time.sleep(5) @@ -108,12 +108,13 @@ class Repository(base.Base): if (timeout_count == 0): break _tag = self.get_tag(repo_name, tag, **kwargs) - if _tag.name == tag and _tag.scan_overview !=None: - if _tag.scan_overview.scan_status == expected_scan_status: - return + if _tag.name == tag and _tag.scan_overview != None: + for report in _tag.scan_overview.values(): + if report.get('scan_status') == expected_scan_status: + return raise Exception("Scan image result is not as expected {}.".format(expected_scan_status)) - def scan_image(self, repo_name, tag, expect_status_code = 200, **kwargs): + def scan_image(self, repo_name, tag, expect_status_code = 202, **kwargs): client = self._get_client(**kwargs) data, status_code, _ = client.repositories_repo_name_tags_tag_scan_post_with_http_info(repo_name, tag) base._assert_status_code(expect_status_code, status_code) diff --git a/tests/apitests/python/test_scan_all_images.py b/tests/apitests/python/test_scan_all_images.py index b49008a35..1d933035b 100644 --- a/tests/apitests/python/test_scan_all_images.py +++ b/tests/apitests/python/test_scan_all_images.py @@ -93,8 +93,8 @@ class TestProjects(unittest.TestCase): self.system.scan_now(**ADMIN_CLIENT) #5. Check if image in project_Alice and another image in project_Luca were both scanned. - self.repo.check_image_scan_result(TestProjects.repo_Alice_name, tag_Alice, expected_scan_status = "finished", **USER_ALICE_CLIENT) - self.repo.check_image_scan_result(TestProjects.repo_Luca_name, tag_Luca, expected_scan_status = "finished", **USER_LUCA_CLIENT) + self.repo.check_image_scan_result(TestProjects.repo_Alice_name, tag_Alice, **USER_ALICE_CLIENT) + self.repo.check_image_scan_result(TestProjects.repo_Luca_name, tag_Luca, **USER_LUCA_CLIENT) if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/apitests/python/test_scan_image.py b/tests/apitests/python/test_scan_image.py index b45e1112f..8f98172c7 100644 --- a/tests/apitests/python/test_scan_image.py +++ b/tests/apitests/python/test_scan_image.py @@ -80,7 +80,7 @@ class TestProjects(unittest.TestCase): #6. Send scan image command and get tag(TA) information to check scan result, it should be finished; self.repo.scan_image(TestProjects.repo_name, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT) - self.repo.check_image_scan_result(TestProjects.repo_name, tag, expected_scan_status = "finished", **TestProjects.USER_SCAN_IMAGE_CLIENT) + self.repo.check_image_scan_result(TestProjects.repo_name, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT) if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/hostcfg.sh b/tests/hostcfg.sh index 9aae9c3a0..ac8bc8966 100755 --- a/tests/hostcfg.sh +++ b/tests/hostcfg.sh @@ -6,4 +6,8 @@ sudo sed "s/reg.mydomain.com/$IP/" -i make/harbor.yml echo "https:" >> make/harbor.yml echo " certificate: /data/cert/server.crt" >> make/harbor.yml -echo " private_key: /data/cert/server.key" >> make/harbor.yml \ No newline at end of file +echo " private_key: /data/cert/server.key" >> make/harbor.yml + +# TODO: remove it when scanner adapter support internal access of harbor +echo "storage_service:" >> make/harbor.yml +echo " ca_bundle: /data/cert/server.crt" >> make/harbor.yml diff --git a/tests/robot-cases/Group0-BAT/API_DB.robot b/tests/robot-cases/Group0-BAT/API_DB.robot index f955e281e..c2ee1d071 100644 --- a/tests/robot-cases/Group0-BAT/API_DB.robot +++ b/tests/robot-cases/Group0-BAT/API_DB.robot @@ -29,18 +29,16 @@ Test Case - Add Replication Rule Harbor API Test ./tests/apitests/python/test_add_replication_rule.py Test Case - Edit Project Creation Harbor API Test ./tests/apitests/python/test_edit_project_creation.py -*** Enable this case after deployment change PR merged *** -*** Test Case - Scan Image *** -*** Harbor API Test ./tests/apitests/python/test_scan_image.py *** +Test Case - Scan Image + Harbor API Test ./tests/apitests/python/test_scan_image.py Test Case - Manage Project Member Harbor API Test ./tests/apitests/python/test_manage_project_member.py Test Case - Project Level Policy Content Trust Harbor API Test ./tests/apitests/python/test_project_level_policy_content_trust.py Test Case - User View Logs Harbor API Test ./tests/apitests/python/test_user_view_logs.py -*** Enable this case after deployment change PR merged *** -*** Test Case - Scan All Images *** -*** Harbor API Test ./tests/apitests/python/test_scan_all_images.py *** +Test Case - Scan All Images + Harbor API Test ./tests/apitests/python/test_scan_all_images.py Test Case - List Helm Charts Harbor API Test ./tests/apitests/python/test_list_helm_charts.py Test Case - Assign Sys Admin