Merge pull request #9434 from heww/clair-adapter

build(clair): internal clair adapter when install with clair
This commit is contained in:
Wenkai Yin(尹文开) 2019-10-17 16:06:10 +08:00 committed by GitHub
commit 97ddff2ac8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 270 additions and 79 deletions

View File

@ -106,6 +106,7 @@ CLAIRDBVERSION=$(VERSIONTAG)
MIGRATORVERSION=$(VERSIONTAG)
REDISVERSION=$(VERSIONTAG)
NOTARYMIGRATEVERSION=v3.5.4
CLAIRADAPTERVERSION=c7db8b15
# version of chartmuseum
CHARTMUSEUMVERSION=v0.9.0
@ -115,6 +116,7 @@ VERSION_TAG: $(VERSIONTAG)
REGISTRY_VERSION: $(REGISTRYVERSION)
NOTARY_VERSION: $(NOTARYVERSION)
CLAIR_VERSION: $(CLAIRVERSION)
CLAIR_ADAPTER_VERSION: $(CLAIRADAPTERVERSION)
CHARTMUSEUM_VERSION: $(CHARTMUSEUMVERSION)
endef
@ -251,7 +253,7 @@ ifeq ($(NOTARYFLAG), true)
DOCKERSAVE_PARA+= goharbor/notary-server-photon:$(NOTARYVERSION)-$(VERSIONTAG) goharbor/notary-signer-photon:$(NOTARYVERSION)-$(VERSIONTAG)
endif
ifeq ($(CLAIRFLAG), true)
DOCKERSAVE_PARA+= goharbor/clair-photon:$(CLAIRVERSION)-$(VERSIONTAG)
DOCKERSAVE_PARA+= goharbor/clair-photon:$(CLAIRVERSION)-$(VERSIONTAG) goharbor/clair-adapter-photon:$(CLAIRADAPTERVERSION)-$(VERSIONTAG)
endif
ifeq ($(MIGRATORFLAG), true)
DOCKERSAVE_PARA+= goharbor/harbor-migrator:$(MIGRATORVERSION)
@ -305,7 +307,7 @@ prepare: update_prepare_version
build:
make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) \
-e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
-e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \
-e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRADAPTERVERSION=$(CLAIRADAPTERVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \
-e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \
-e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER) \
-e NPM_REGISTRY=$(NPM_REGISTRY)

View File

@ -4008,7 +4008,7 @@ paths:
schema:
type: array
items:
$ref: '#/definitions/ImmutableTagRule'
$ref: '#/definitions/ImmutableTagRule'
'400':
description: Illegal format of provided ID value.
'401':
@ -5115,35 +5115,7 @@ definitions:
description: 'The signature of image, defined by RepoSignature. If it is null, the image is unsigned.'
scan_overview:
type: object
description: The overview of the scan result. This is an optional property.
properties:
digest:
type: string
description: The digest of the image.
scan_status:
type: string
description: 'The status of the scan job, it can be "pending", "running", "finished", "error".'
job_id:
type: integer
description: The ID of the job on jobservice to scan the image.
severity:
type: integer
description: '0-Not scanned, 1-Negligible, 2-Unknown, 3-Low, 4-Medium, 5-High'
details_key:
type: string
description: 'The top layer name of this image in Clair, this is for calling Clair API to get the vulnerability list of this image.'
components:
type: object
description: The components overview of the image.
properties:
total:
type: integer
description: Total number of the components in this image.
summary:
description: List of number of components of different severities.
type: array
items:
$ref: '#/definitions/ComponentOverviewEntry'
description: The overview of the scan result.
labels:
type: array
description: The label list.
@ -6274,12 +6246,10 @@ definitions:
$ref: '#/definitions/RetentionRule'
trigger:
type: object
items:
$ref: '#/definitions/RetentionRuleTrigger'
$ref: '#/definitions/RetentionRuleTrigger'
scope:
type: object
items:
$ref: '#/definitions/RetentionPolicyScope'
$ref: '#/definitions/RetentionPolicyScope'
RetentionRuleTrigger:
type: object

View File

@ -64,6 +64,10 @@ DOCKERFILEPATH_CLAIR=$(DOCKERFILEPATH)/clair
DOCKERFILENAME_CLAIR=Dockerfile
DOCKERIMAGENAME_CLAIR=goharbor/clair-photon
DOCKERFILEPATH_CLAIR_ADAPTER=$(DOCKERFILEPATH)/clair-adapter
DOCKERFILENAME_CLAIR_ADAPTER=Dockerfile
DOCKERIMAGENAME_CLAIR_ADAPTER=goharbor/clair-adapter-photon
DOCKERFILEPATH_NGINX=$(DOCKERFILEPATH)/nginx
DOCKERFILENAME_NGINX=Dockerfile
DOCKERIMAGENAME_NGINX=goharbor/nginx-photon
@ -141,6 +145,16 @@ _build_clair:
echo "Done." ; \
fi
_build_clair_adapter:
# TODO: add support to fetch clair adapter binary from google storage ranther than build from source
@if [ "$(CLAIRFLAG)" = "true" ] ; then \
cd $(DOCKERFILEPATH_CLAIR_ADAPTER) && $(DOCKERFILEPATH_CLAIR_ADAPTER)/builder $(CLAIRADAPTERVERSION) && cd - ; \
echo "building clair adapter container for photon..." ; \
$(DOCKERBUILD) -f $(DOCKERFILEPATH_CLAIR_ADAPTER)/$(DOCKERFILENAME_CLAIR_ADAPTER) -t $(DOCKERIMAGENAME_CLAIR_ADAPTER):$(CLAIRADAPTERVERSION)-$(VERSIONTAG) . ; \
rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary; \
echo "Done." ; \
fi
_build_chart_server:
@if [ "$(CHARTFLAG)" = "true" ] ; then \
if [ "$(BUILDBIN)" != "true" ] ; then \
@ -209,7 +223,7 @@ define _get_binary
$(WGET) --timeout 30 --no-check-certificate $1 -O $2
endef
build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_redis _build_migrator _build_chart_server
build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_clair_adapter _build_redis _build_migrator _build_chart_server
cleanimage:
@echo "cleaning image for photon..."

View File

@ -0,0 +1,20 @@
FROM photon:2.0
RUN tdnf install -y sudo >>/dev/null\
&& tdnf clean all \
&& mkdir /clair-adapter/ \
&& groupadd -r -g 10000 clair-adapter \
&& useradd --no-log-init -m -r -g 10000 -u 10000 clair-adapter
COPY ./make/photon/clair-adapter/binary/harbor-scanner-clair /clair-adapter/clair-adapter
RUN chown -R 10000:10000 /clair-adapter \
&& chmod u+x /clair-adapter/clair-adapter
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS 127.0.0.1:8080/healthy || exit 1
USER clair-adapter
ENTRYPOINT ["/clair-adapter/clair-adapter"]

View File

@ -0,0 +1,7 @@
FROM golang:1.12.5
ADD . /go/src/github.com/goharbor/harbor-scanner-clair/
WORKDIR /go/src/github.com/goharbor/harbor-scanner-clair/
RUN export GOOS=linux GO111MODULE=on CGO_ENABLED=0 && \
go build github.com/goharbor/harbor-scanner-clair/cmd/harbor-scanner-clair

View File

@ -0,0 +1,39 @@
#!/bin/bash
set +e
if [ -z $1 ]; then
error "Please set the 'version' variable"
exit 1
fi
VERSION="$1"
set -e
# the temp folder to store binary file...
mkdir -p binary
rm -rf binary/harbor-scanner-clair || true
cd `dirname $0`
cur=$PWD
# the temp folder to store distribution source code...
TEMP=`mktemp -d ${TMPDIR-/tmp}/clair-adapter.XXXXXX`
git clone https://github.com/danielpacak/harbor-scanner-clair.git $TEMP
cd $TEMP; git checkout $VERSION; cd -
echo 'build the clair adapter binary bases on the golang:1.12.5...'
cp Dockerfile.binary $TEMP
docker build -f $TEMP/Dockerfile.binary -t clair-adapter-golang $TEMP
echo 'copy the clair adapter binary to local...'
ID=$(docker create clair-adapter-golang)
docker cp $ID:/go/src/github.com/goharbor/harbor-scanner-clair/harbor-scanner-clair binary
docker rm -f $ID
docker rmi -f clair-adapter-golang
echo "Build clair adapter binary success, then to build photon image..."
cd $cur
rm -rf $TEMP

View File

@ -13,6 +13,7 @@ from utils.core import prepare_core
from utils.notary import prepare_notary
from utils.log import prepare_log_configs
from utils.clair import prepare_clair
from utils.clair_adapter import prepare_clair_adapter
from utils.chart import prepare_chartmuseum
from utils.docker_compose import prepare_docker_compose
from utils.nginx import prepare_nginx, nginx_confd_dir
@ -54,6 +55,7 @@ def main(conf, with_notary, with_clair, with_chartmuseum):
if with_clair:
prepare_clair(config_dict)
prepare_clair_adapter(config_dict)
if with_chartmuseum:
prepare_chartmuseum(config_dict)

View File

@ -0,0 +1 @@
SCANNER_CLAIR_URL={{clair_url}}

View File

@ -36,6 +36,7 @@ CORE_URL={{core_url}}
CORE_LOCAL_URL={{core_local_url}}
JOBSERVICE_URL={{jobservice_url}}
CLAIR_URL={{clair_url}}
CLAIR_ADAPTER_URL={{clair_adapter_url}}
NOTARY_URL={{notary_url}}
REGISTRY_STORAGE_PROVIDER_NAME={{storage_provider_name}}
READ_ONLY=false

View File

@ -56,7 +56,7 @@ services:
- log
logging:
driver: "syslog"
options:
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "registry"
registryctl:
@ -89,7 +89,7 @@ services:
- log
logging:
driver: "syslog"
options:
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "registryctl"
{% if external_database == False %}
@ -125,7 +125,7 @@ services:
- log
logging:
driver: "syslog"
options:
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "postgresql"
{% endif %}
@ -186,7 +186,7 @@ services:
{% endif %}
logging:
driver: "syslog"
options:
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "core"
portal:
@ -307,7 +307,7 @@ services:
- log
logging:
driver: "syslog"
options:
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "proxy"
{% if with_notary %}
@ -336,7 +336,7 @@ services:
- notary-signer
logging:
driver: "syslog"
options:
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "notary-server"
notary-signer:
@ -366,7 +366,7 @@ services:
{% endif %}
logging:
driver: "syslog"
options:
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "notary-signer"
{% endif %}
@ -406,6 +406,29 @@ services:
tag: "clair"
env_file:
./common/config/clair/clair_env
clair-adapter:
networks:
- harbor-clair
container_name: clair-adapter
image: goharbor/clair-adapter-photon:{{clair_adapter_version}}
restart: always
cap_drop:
- ALL
cap_add:
- DAC_OVERRIDE
- SETGID
- SETUID
cpu_quota: 50000
dns_search: .
depends_on:
- clair
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "clair-adapter"
env_file:
./common/config/clair-adapter/env
{% endif %}
{% if with_chartmuseum %}
chartmuseum:
@ -439,7 +462,7 @@ services:
{% endif %}
logging:
driver: "syslog"
options:
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "chartmuseum"
env_file:

View File

@ -0,0 +1,18 @@
import os
from g import templates_dir, config_dir
from .jinja import render_jinja
from .misc import prepare_dir
clair_adapter_template_dir = os.path.join(templates_dir, "clair-adapter")
def prepare_clair_adapter(config_dict):
clair_adapter_config_dir = prepare_dir(config_dir, "clair-adapter")
clair_adapter_env_path = os.path.join(clair_adapter_config_dir, "env")
clair_adapter_env_template = os.path.join(clair_adapter_template_dir, "env.jinja")
render_jinja(
clair_adapter_env_template,
clair_adapter_env_path,
**config_dict)

View File

@ -74,6 +74,7 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_chartmuseu
'token_service_url': "http://core:8080/service/token",
'jobservice_url': 'http://jobservice:8080',
'clair_url': 'http://clair:6060',
'clair_adapter_url': 'http://clair-adapter:8080',
'notary_url': 'http://notary-server:4443',
'chart_repository_url': 'http://chartmuseum:9999'
}

View File

@ -11,9 +11,10 @@ docker_compose_yml_path = '/compose_location/docker-compose.yml'
def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):
versions = parse_versions()
VERSION_TAG = versions.get('VERSION_TAG') or 'dev'
REGISTRY_VERSION = versions.get('REGISTRY_VERSION') or 'v2.7.1'
REGISTRY_VERSION = versions.get('REGISTRY_VERSION') or 'v2.7.1-patch-2819-2553'
NOTARY_VERSION = versions.get('NOTARY_VERSION') or 'v0.6.1'
CLAIR_VERSION = versions.get('CLAIR_VERSION') or 'v2.0.9'
CLAIR_ADAPTER_VERSION = versions.get('CLAIR_ADAPTER_VERSION') or ''
CHARTMUSEUM_VERSION = versions.get('CHARTMUSEUM_VERSION') or 'v0.9.0'
rendering_variables = {
@ -22,6 +23,7 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):
'redis_version': VERSION_TAG,
'notary_version': '{}-{}'.format(NOTARY_VERSION, VERSION_TAG),
'clair_version': '{}-{}'.format(CLAIR_VERSION, VERSION_TAG),
'clair_adapter_version': '{}-{}'.format(CLAIR_ADAPTER_VERSION, VERSION_TAG),
'chartmuseum_version': '{}-{}'.format(CHARTMUSEUM_VERSION, VERSION_TAG),
'data_volume': configs['data_volume'],
'log_location': configs['log_location'],

View File

@ -73,6 +73,7 @@ var (
{Name: common.ClairDBSSLMode, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_DB_SSLMODE", DefaultValue: "disable", ItemType: &StringType{}, Editable: false},
{Name: common.ClairDBUsername, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_DB_USERNAME", DefaultValue: "postgres", ItemType: &StringType{}, Editable: false},
{Name: common.ClairURL, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_URL", DefaultValue: "http://clair:6060", ItemType: &StringType{}, Editable: false},
{Name: common.ClairAdapterURL, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_ADAPTER_URL", DefaultValue: "http://clair-adapter:8080", ItemType: &StringType{}, Editable: false},
{Name: common.CoreURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "CORE_URL", DefaultValue: "http://core:8080", ItemType: &StringType{}, Editable: false},
{Name: common.CoreLocalURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "CORE_LOCAL_URL", DefaultValue: "http://127.0.0.1:8080", ItemType: &StringType{}, Editable: false},

View File

@ -121,6 +121,7 @@ const (
GroupMember = "g"
ReadOnly = "read_only"
ClairURL = "clair_url"
ClairAdapterURL = "clair_adapter_url"
NotaryURL = "notary_url"
DefaultCoreEndpoint = "http://core:8080"
DefaultNotaryEndpoint = "http://notary-server:4443"

View File

@ -386,6 +386,11 @@ func ClairDB() (*models.PostGreSQL, error) {
return clairDB, nil
}
// ClairAdapterEndpoint returns the endpoint of clair adapter instance, by default it's the one deployed within Harbor.
func ClairAdapterEndpoint() string {
return cfgMgr.Get(common.ClairAdapterURL).GetString()
}
// AdmiralEndpoint returns the URL of admiral, if Harbor is not deployed with admiral it should return an empty string.
func AdmiralEndpoint() string {
if cfgMgr.Get(common.AdmiralEndpoint).GetString() == "NA" {

View File

@ -48,6 +48,8 @@ import (
_ "github.com/goharbor/harbor/src/core/notifier/topic"
"github.com/goharbor/harbor/src/core/service/token"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/pkg/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/types"
"github.com/goharbor/harbor/src/replication"
@ -215,6 +217,19 @@ func main() {
if err := dao.InitClairDB(clairDB); err != nil {
log.Fatalf("failed to initialize clair database: %v", err)
}
// TODO: change to be internal adapter
reg := &scanner.Registration{
Name: "Clair",
Description: "The clair scanner adapter",
URL: config.ClairAdapterEndpoint(),
Disabled: false,
IsDefault: true,
}
if err := scan.EnsureScanner(reg); err != nil {
log.Fatalf("failed to initialize clair scanner: %v", err)
}
}
closing := make(chan struct{})

View File

@ -15,10 +15,10 @@
package scan
import (
"encoding/base64"
"fmt"
"time"
"github.com/goharbor/harbor/src/common"
cj "github.com/goharbor/harbor/src/common/job"
jm "github.com/goharbor/harbor/src/common/job/models"
"github.com/goharbor/harbor/src/common/rbac"
@ -102,6 +102,14 @@ func NewController() Controller {
}
}
func (bc *basicController) jobClient() cj.Client {
if bc.jc == nil {
return cj.GlobalClient
}
return bc.jc
}
// Scan ...
func (bc *basicController) Scan(artifact *v1.Artifact) error {
if artifact == nil {
@ -276,7 +284,7 @@ func (bc *basicController) GetScanLog(uuid string) ([]byte, error) {
}
// Job log
return bc.jc.GetJobLog(sr.JobID)
return bc.jobClient().GetJobLog(sr.JobID)
}
// HandleJobHooks ...
@ -321,8 +329,8 @@ func (bc *basicController) HandleJobHooks(trackID string, change *job.StatusChan
return bc.manager.UpdateStatus(trackID, change.Status, change.Metadata.Revision)
}
// makeRobotAccount creates a robot account based on the arguments for scanning.
func (bc *basicController) makeRobotAccount(pid int64, repository string, ttl int64) (string, error) {
// makeAuthorization creates authorization from a robot account based on the arguments for scanning.
func (bc *basicController) makeAuthorization(pid int64, repository string, ttl int64) (string, error) {
// Use uuid as name to avoid duplicated entries.
UUID, err := bc.uuid()
if err != nil {
@ -333,25 +341,28 @@ func (bc *basicController) makeRobotAccount(pid int64, repository string, ttl in
logger.Warningf("repository %s and expire time %d are not supported by robot controller", repository, expireAt)
resource := fmt.Sprintf("/project/%d/repository", pid)
resource := rbac.NewProjectNamespace(pid).Resource(rbac.ResourceRepository)
access := []*rbac.Policy{{
Resource: rbac.Resource(resource),
Action: "pull",
Resource: resource,
Action: rbac.ActionPull,
}}
account := &model.RobotCreate{
Name: fmt.Sprintf("%s%s", common.RobotPrefix, UUID),
robotReq := &model.RobotCreate{
Name: UUID,
Description: "for scan",
ProjectID: pid,
Access: access,
}
rb, err := bc.rc.CreateRobotAccount(account)
rb, err := bc.rc.CreateRobotAccount(robotReq)
if err != nil {
return "", errors.Wrap(err, "scan controller: make robot account")
}
return rb.Token, nil
username := rb.Name
password := rb.Token
return "Basic " + base64.StdEncoding.EncodeToString([]byte(username+":"+password)), nil
}
// launchScanJob launches a job to run scan
@ -361,8 +372,8 @@ func (bc *basicController) launchScanJob(trackID string, artifact *v1.Artifact,
return "", errors.Wrap(err, "scan controller: launch scan job")
}
// Make a robot account with 30 minutes
robotAccount, err := bc.makeRobotAccount(artifact.NamespaceID, artifact.Repository, 1800)
// Make authorization from a robot account with 30 minutes
authorization, err := bc.makeAuthorization(artifact.NamespaceID, artifact.Repository, 1800)
if err != nil {
return "", errors.Wrap(err, "scan controller: launch scan job")
}
@ -371,7 +382,7 @@ func (bc *basicController) launchScanJob(trackID string, artifact *v1.Artifact,
scanReq := &v1.ScanRequest{
Registry: &v1.Registry{
URL: externalURL,
Authorization: robotAccount,
Authorization: authorization,
},
Artifact: artifact,
}
@ -407,5 +418,5 @@ func (bc *basicController) launchScanJob(trackID string, artifact *v1.Artifact,
StatusHook: hookURL,
}
return bc.jc.SubmitJob(j)
return bc.jobClient().SubmitJob(j)
}

View File

@ -15,6 +15,7 @@
package scan
import (
"encoding/base64"
"encoding/json"
"fmt"
"testing"
@ -164,7 +165,7 @@ func (suite *ControllerTestSuite) SetupSuite() {
Action: "pull",
}}
rname := fmt.Sprintf("%s%s", common.RobotPrefix, "the-uuid-123")
rname := "the-uuid-123"
account := &model.RobotCreate{
Name: rname,
Description: "for scan",
@ -173,7 +174,7 @@ func (suite *ControllerTestSuite) SetupSuite() {
}
rc.On("CreateRobotAccount", account).Return(&model.Robot{
ID: 1,
Name: rname,
Name: common.RobotPrefix + rname,
Token: "robot-account",
Description: "for scan",
ProjectID: suite.artifact.NamespaceID,
@ -183,7 +184,7 @@ func (suite *ControllerTestSuite) SetupSuite() {
req := &v1.ScanRequest{
Registry: &v1.Registry{
URL: "https://core.com",
Authorization: "robot-account",
Authorization: "Basic " + base64.StdEncoding.EncodeToString([]byte(common.RobotPrefix+"the-uuid-123:robot-account")),
},
Artifact: suite.artifact,
}

44
src/pkg/scan/init.go Normal file
View File

@ -0,0 +1,44 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/q"
sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
)
// EnsureScanner ensure the scanner which specially name exists in the system
func EnsureScanner(registration *scanner.Registration) error {
q := &q.Query{
Keywords: map[string]interface{}{"url": registration.URL},
}
registrations, err := sc.DefaultController.ListRegistrations(q)
if err != nil {
return err
}
if len(registrations) == 0 {
if _, err := sc.DefaultController.CreateRegistration(registration); err != nil {
return err
}
log.Infof("initialized scanner named %s", registration.Name)
}
return nil
}

View File

@ -3,6 +3,10 @@
import sys
import time
import swagger_client
try:
from urllib import getproxies
except ImportError:
from urllib.request import getproxies
class Server:
def __init__(self, endpoint, verify_ssl):
@ -23,6 +27,12 @@ def _create_client(server, credential, debug):
cfg.username = credential.username
cfg.password = credential.password
cfg.debug = debug
proxies = getproxies()
proxy = proxies.get('http', proxies.get('all', None))
if proxy:
cfg.proxy = proxy
return swagger_client.ProductsApi(swagger_client.ApiClient(cfg))
def _assert_status_code(expect_code, return_code):

View File

@ -100,7 +100,7 @@ class Repository(base.Base):
if tag.scan_overview != None:
raise Exception("Image should be <Not Scanned> state!")
def check_image_scan_result(self, repo_name, tag, expected_scan_status = "finished", **kwargs):
def check_image_scan_result(self, repo_name, tag, expected_scan_status = "Success", **kwargs):
timeout_count = 30
while True:
time.sleep(5)
@ -108,12 +108,13 @@ class Repository(base.Base):
if (timeout_count == 0):
break
_tag = self.get_tag(repo_name, tag, **kwargs)
if _tag.name == tag and _tag.scan_overview !=None:
if _tag.scan_overview.scan_status == expected_scan_status:
return
if _tag.name == tag and _tag.scan_overview != None:
for report in _tag.scan_overview.values():
if report.get('scan_status') == expected_scan_status:
return
raise Exception("Scan image result is not as expected {}.".format(expected_scan_status))
def scan_image(self, repo_name, tag, expect_status_code = 200, **kwargs):
def scan_image(self, repo_name, tag, expect_status_code = 202, **kwargs):
client = self._get_client(**kwargs)
data, status_code, _ = client.repositories_repo_name_tags_tag_scan_post_with_http_info(repo_name, tag)
base._assert_status_code(expect_status_code, status_code)

View File

@ -93,8 +93,8 @@ class TestProjects(unittest.TestCase):
self.system.scan_now(**ADMIN_CLIENT)
#5. Check if image in project_Alice and another image in project_Luca were both scanned.
self.repo.check_image_scan_result(TestProjects.repo_Alice_name, tag_Alice, expected_scan_status = "finished", **USER_ALICE_CLIENT)
self.repo.check_image_scan_result(TestProjects.repo_Luca_name, tag_Luca, expected_scan_status = "finished", **USER_LUCA_CLIENT)
self.repo.check_image_scan_result(TestProjects.repo_Alice_name, tag_Alice, **USER_ALICE_CLIENT)
self.repo.check_image_scan_result(TestProjects.repo_Luca_name, tag_Luca, **USER_LUCA_CLIENT)
if __name__ == '__main__':
unittest.main()

View File

@ -80,7 +80,7 @@ class TestProjects(unittest.TestCase):
#6. Send scan image command and get tag(TA) information to check scan result, it should be finished;
self.repo.scan_image(TestProjects.repo_name, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT)
self.repo.check_image_scan_result(TestProjects.repo_name, tag, expected_scan_status = "finished", **TestProjects.USER_SCAN_IMAGE_CLIENT)
self.repo.check_image_scan_result(TestProjects.repo_name, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT)
if __name__ == '__main__':
unittest.main()

View File

@ -6,4 +6,8 @@ sudo sed "s/reg.mydomain.com/$IP/" -i make/harbor.yml
echo "https:" >> make/harbor.yml
echo " certificate: /data/cert/server.crt" >> make/harbor.yml
echo " private_key: /data/cert/server.key" >> make/harbor.yml
echo " private_key: /data/cert/server.key" >> make/harbor.yml
# TODO: remove it when scanner adapter support internal access of harbor
echo "storage_service:" >> make/harbor.yml
echo " ca_bundle: /data/cert/server.crt" >> make/harbor.yml

View File

@ -29,18 +29,16 @@ Test Case - Add Replication Rule
Harbor API Test ./tests/apitests/python/test_add_replication_rule.py
Test Case - Edit Project Creation
Harbor API Test ./tests/apitests/python/test_edit_project_creation.py
*** Enable this case after deployment change PR merged ***
*** Test Case - Scan Image ***
*** Harbor API Test ./tests/apitests/python/test_scan_image.py ***
Test Case - Scan Image
Harbor API Test ./tests/apitests/python/test_scan_image.py
Test Case - Manage Project Member
Harbor API Test ./tests/apitests/python/test_manage_project_member.py
Test Case - Project Level Policy Content Trust
Harbor API Test ./tests/apitests/python/test_project_level_policy_content_trust.py
Test Case - User View Logs
Harbor API Test ./tests/apitests/python/test_user_view_logs.py
*** Enable this case after deployment change PR merged ***
*** Test Case - Scan All Images ***
*** Harbor API Test ./tests/apitests/python/test_scan_all_images.py ***
Test Case - Scan All Images
Harbor API Test ./tests/apitests/python/test_scan_all_images.py
Test Case - List Helm Charts
Harbor API Test ./tests/apitests/python/test_list_helm_charts.py
Test Case - Assign Sys Admin