From ae2d868fd43575fa6bd25a9a006bd495c180bcc9 Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Tue, 13 Jun 2017 21:46:52 +0800 Subject: [PATCH] handlers for image scan, store results overview in DB --- make/common/db/registry.sql | 14 +++ make/common/db/registry_sqlite.sql | 13 ++ make/common/templates/clair/config.yaml | 2 +- make/docker-compose.clair.yml | 3 + src/common/dao/dao_test.go | 43 +++++++ src/common/dao/scan_job.go | 67 ++++++++++ src/common/models/base.go | 3 +- src/common/models/scan_job.go | 46 +++++++ src/common/models/token.go | 22 ++++ src/common/utils/clair/client.go | 108 ++++++++++++++++ src/common/utils/clair/utils.go | 37 ++++++ src/common/utils/clair/utils_test.go | 35 ++++++ src/common/utils/log/logger.go | 5 + .../utils/registry/auth/tokenauthorizer.go | 7 +- src/jobservice/config/config.go | 5 + src/jobservice/job/job_test.go | 8 +- src/jobservice/job/jobs.go | 4 + src/jobservice/job/statemachine.go | 5 +- src/jobservice/scan/context.go | 15 +-- src/jobservice/scan/handlers.go | 116 +++++++++++++++++- src/jobservice/utils/utils.go | 50 +++++++- src/ui/service/token/authutils.go | 20 ++- src/ui/service/token/creator.go | 5 +- src/ui/service/token/token_test.go | 4 +- .../github.com/astaxie/beego/orm/orm.go | 3 +- 25 files changed, 605 insertions(+), 35 deletions(-) create mode 100644 src/common/models/token.go create mode 100644 src/common/utils/clair/client.go create mode 100644 src/common/utils/clair/utils.go create mode 100644 src/common/utils/clair/utils_test.go diff --git a/make/common/db/registry.sql b/make/common/db/registry.sql index 62d301b6d..2c4a8ab04 100644 --- a/make/common/db/registry.sql +++ b/make/common/db/registry.sql @@ -180,6 +180,20 @@ create table img_scan_job ( PRIMARY KEY (id) ); +create table img_scan_overview ( + image_digest varchar(128) NOT NULL, + scan_job_id int NOT NULL, + /* 0 indicates none, the higher the number, the more severe the status */ + severity int NOT NULL default 0, + /* the json string to store components severity status, currently use a json to be more flexible and avoid creating additional tables. */ + components_overview varchar(2048), + /* primary key for querying details, in clair it should be the name of the "top layer" */ + details_key varchar(128), + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, + PRIMARY KEY(image_digest) + ); + create table properties ( k varchar(64) NOT NULL, v varchar(128) NOT NULL, diff --git a/make/common/db/registry_sqlite.sql b/make/common/db/registry_sqlite.sql index d121a9bae..fecc00610 100644 --- a/make/common/db/registry_sqlite.sql +++ b/make/common/db/registry_sqlite.sql @@ -171,6 +171,19 @@ create table img_scan_job ( update_time timestamp default CURRENT_TIMESTAMP ); +create table img_scan_overview ( + image_digest varchar(128) PRIMARY KEY, + scan_job_id int NOT NULL, + /* 0 indicates none, the higher the number, the more severe the status */ + severity int NOT NULL default 0, + /* the json string to store components severity status, currently use a json to be more flexible and avoid creating additional tables. */ + components_overview varchar(2048), + /* primary key for querying details, in clair it should be the name of the "top layer" */ + details_key varchar(128), + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP + ); + CREATE INDEX policy ON replication_job (policy_id); CREATE INDEX poid_uptime ON replication_job (policy_id, update_time); diff --git a/make/common/templates/clair/config.yaml b/make/common/templates/clair/config.yaml index b09a0870f..f93129e4a 100644 --- a/make/common/templates/clair/config.yaml +++ b/make/common/templates/clair/config.yaml @@ -16,7 +16,7 @@ clair: # Deadline before an API request will respond with a 503 timeout: 300s updater: - interval: 0h + interval: 2h notifier: attempts: 3 diff --git a/make/docker-compose.clair.yml b/make/docker-compose.clair.yml index b6e5b46af..c90b4fb9d 100644 --- a/make/docker-compose.clair.yml +++ b/make/docker-compose.clair.yml @@ -8,6 +8,9 @@ services: jobservice: networks: - harbor-clair + registry: + networks: + - harbor-clair postgres: networks: harbor-clair: diff --git a/src/common/dao/dao_test.go b/src/common/dao/dao_test.go index 7114f6a98..175c04eb5 100644 --- a/src/common/dao/dao_test.go +++ b/src/common/dao/dao_test.go @@ -134,6 +134,12 @@ const publicityOn = 1 const publicityOff = 0 func TestMain(m *testing.M) { + orm.Debug = true + f, err := os.Create("/root/jtdbtest.out") + if err != nil { + panic(err) + } + orm.DebugLog = orm.NewLog(f) databases := []string{"mysql", "sqlite"} for _, database := range databases { log.Infof("run test cases for database: %s", database) @@ -1693,3 +1699,40 @@ func TestUpdateScanJobStatus(t *testing.T) { err = ClearTable(models.ScanJobTable) assert.Nil(err) } + +func TestImgScanOverview(t *testing.T) { + assert := assert.New(t) + err := ClearTable(models.ScanOverviewTable) + assert.Nil(err) + digest := "sha256:0204dc6e09fa57ab99ac40e415eb637d62c8b2571ecbbc9ca0eb5e2ad2b5c56f" + res, err := GetImgScanOverview(digest) + assert.Nil(err) + assert.Nil(res) + err = SetScanJobForImg(digest, 33) + assert.Nil(err) + res, err = GetImgScanOverview(digest) + assert.Nil(err) + assert.Equal(int64(33), res.JobID) + err = SetScanJobForImg(digest, 22) + assert.Nil(err) + res, err = GetImgScanOverview(digest) + assert.Nil(err) + assert.Equal(int64(22), res.JobID) + pk := "22-sha256:sdfsdfarfwefwr23r43t34ggregergerger" + comp := &models.ComponentsOverview{ + Total: 2, + Summary: []*models.ComponentsOverviewEntry{ + &models.ComponentsOverviewEntry{ + Sev: int(models.SevMedium), + Count: 2, + }, + }, + } + err = UpdateImgScanOverview(digest, pk, models.SevMedium, comp) + assert.Nil(err) + res, err = GetImgScanOverview(digest) + assert.Nil(err) + assert.Equal(pk, res.DetailsKey) + assert.Equal(int(models.SevMedium), res.Sev) + assert.Equal(2, res.CompOverview.Summary[0].Count) +} diff --git a/src/common/dao/scan_job.go b/src/common/dao/scan_job.go index 945c5af17..51f9ffb68 100644 --- a/src/common/dao/scan_job.go +++ b/src/common/dao/scan_job.go @@ -18,6 +18,7 @@ import ( "github.com/astaxie/beego/orm" "github.com/vmware/harbor/src/common/models" + "encoding/json" "fmt" "time" ) @@ -79,3 +80,69 @@ func scanJobQs(limit ...int) orm.QuerySeter { } return o.QueryTable(models.ScanJobTable).Limit(l) } + +// SetScanJobForImg updates the scan_job_id based on the digest of image, if there's no data, it created one record. +func SetScanJobForImg(digest string, jobID int64) error { + o := GetOrmer() + rec := &models.ImgScanOverview{ + Digest: digest, + JobID: jobID, + UpdateTime: time.Now(), + } + created, _, err := o.ReadOrCreate(rec, "Digest") + if err != nil { + return err + } + if !created { + rec.JobID = jobID + n, err := o.Update(rec, "JobID", "UpdateTime") + if n == 0 { + return fmt.Errorf("Failed to set scan job for image with digest: %s, error: %v", digest, err) + } + } + return nil +} + +// GetImgScanOverview returns the ImgScanOverview based on the digest. +func GetImgScanOverview(digest string) (*models.ImgScanOverview, error) { + o := GetOrmer() + rec := &models.ImgScanOverview{ + Digest: digest, + } + err := o.Read(rec) + if err != nil && err != orm.ErrNoRows { + return nil, err + } + if err == orm.ErrNoRows { + return nil, nil + } + if len(rec.CompOverviewStr) > 0 { + co := &models.ComponentsOverview{} + if err := json.Unmarshal([]byte(rec.CompOverviewStr), co); err != nil { + return nil, err + } + rec.CompOverview = co + } + return rec, nil +} + +// UpdateImgScanOverview updates the serverity and components status of a record in img_scan_overview +func UpdateImgScanOverview(digest, detailsKey string, sev models.Severity, compOverview *models.ComponentsOverview) error { + o := GetOrmer() + b, err := json.Marshal(compOverview) + if err != nil { + return err + } + rec := &models.ImgScanOverview{ + Digest: digest, + Sev: int(sev), + CompOverviewStr: string(b), + DetailsKey: detailsKey, + UpdateTime: time.Now(), + } + n, err := o.Update(rec, "Sev", "CompOverviewStr", "DetailsKey", "UpdateTime") + if n == 0 || err != nil { + return fmt.Errorf("Failed to update scan overview record with digest: %s, error: %v", digest, err) + } + return nil +} diff --git a/src/common/models/base.go b/src/common/models/base.go index f861399f2..7fec6cb70 100644 --- a/src/common/models/base.go +++ b/src/common/models/base.go @@ -27,5 +27,6 @@ func init() { new(Role), new(AccessLog), new(ScanJob), - new(RepoRecord)) + new(RepoRecord), + new(ImgScanOverview)) } diff --git a/src/common/models/scan_job.go b/src/common/models/scan_job.go index 286429cbc..343ffb233 100644 --- a/src/common/models/scan_job.go +++ b/src/common/models/scan_job.go @@ -19,6 +19,9 @@ import "time" //ScanJobTable is the name of the table whose data is mapped by ScanJob struct. const ScanJobTable = "img_scan_job" +//ScanOverviewTable is the name of the table whose data is mapped by ImgScanOverview struct. +const ScanOverviewTable = "img_scan_overview" + //ScanJob is the model to represent a job for image scan in DB. type ScanJob struct { ID int64 `orm:"pk;auto;column(id)" json:"id"` @@ -30,7 +33,50 @@ type ScanJob struct { UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` } +// Severity represents the severity of a image/component in terms of vulnerability. +type Severity int64 + +// Sevxxx is the list of severity of image after scanning. +const ( + _ Severity = iota + SevNone + SevUnknown + SevLow + SevMedium + SevHigh +) + //TableName is required by by beego orm to map ScanJob to table img_scan_job func (s *ScanJob) TableName() string { return ScanJobTable } + +//ImgScanOverview mapped to a record of image scan overview. +type ImgScanOverview struct { + Digest string `orm:"pk;column(image_digest)" json:"image_digest"` + Status string `orm:"-" json:"scan_status"` + JobID int64 `orm:"column(scan_job_id)" json:"job_id"` + Sev int `orm:"column(severity)" json:"severity"` + CompOverviewStr string `orm:"column(components_overview)" json:"-"` + CompOverview *ComponentsOverview `orm:"-" json:"components"` + DetailsKey string `orm:"column(details_key)" json:"details_key"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` +} + +//TableName ... +func (iso *ImgScanOverview) TableName() string { + return ScanOverviewTable +} + +//ComponentsOverview has the total number and a list of components number of different serverity level. +type ComponentsOverview struct { + Total int `json:"total"` + Summary []*ComponentsOverviewEntry `json:"summary"` +} + +//ComponentsOverviewEntry ... +type ComponentsOverviewEntry struct { + Sev int `json:"severity"` + Count int `json:"count"` +} diff --git a/src/common/models/token.go b/src/common/models/token.go new file mode 100644 index 000000000..f83bec346 --- /dev/null +++ b/src/common/models/token.go @@ -0,0 +1,22 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +// Token represents the json returned by registry token service +type Token struct { + Token string `json:"token"` + ExpiresIn int `json:"expires_in"` + IssuedAt string `json:"issued_at"` +} diff --git a/src/common/utils/clair/client.go b/src/common/utils/clair/client.go new file mode 100644 index 000000000..59c4c95dd --- /dev/null +++ b/src/common/utils/clair/client.go @@ -0,0 +1,108 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clair + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + // "path" + + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" +) + +// Client communicates with clair endpoint to scan image and get detailed scan result +type Client struct { + endpoint string + //need to customize the logger to write output to job log. + logger *log.Logger + client *http.Client +} + +// NewClient creates a new instance of client, set the logger as the job's logger if it's used in a job handler. +func NewClient(endpoint string, logger *log.Logger) *Client { + if logger == nil { + logger = log.DefaultLogger() + } + return &Client{ + endpoint: endpoint, + logger: logger, + client: &http.Client{}, + } +} + +// ScanLayer calls Clair's API to scan a layer. +func (c *Client) ScanLayer(l models.ClairLayer) error { + layer := models.ClairLayerEnvelope{ + Layer: &l, + Error: nil, + } + data, err := json.Marshal(layer) + if err != nil { + return err + } + c.logger.Infof("endpoint: %s", c.endpoint) + c.logger.Infof("body: %s", string(data)) + req, err := http.NewRequest("POST", c.endpoint+"/v1/layers", bytes.NewReader(data)) + if err != nil { + return err + } + req.Header.Set(http.CanonicalHeaderKey("Content-Type"), "application/json") + resp, err := c.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + c.logger.Infof("response code: %d", resp.StatusCode) + if resp.StatusCode != http.StatusCreated { + c.logger.Warningf("Unexpected status code: %d", resp.StatusCode) + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + return fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b)) + } + c.logger.Infof("Returning.") + return nil +} + +// GetResult calls Clair's API to get layers with detailed vulnerability list +func (c *Client) GetResult(layerName string) (*models.ClairLayerEnvelope, error) { + req, err := http.NewRequest("GET", c.endpoint+"/v1/layers/"+layerName+"?features&vulnerabilities", nil) + if err != nil { + return nil, err + } + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b)) + } + var res models.ClairLayerEnvelope + err = json.Unmarshal(b, &res) + if err != nil { + return nil, err + } + return &res, nil +} diff --git a/src/common/utils/clair/utils.go b/src/common/utils/clair/utils.go new file mode 100644 index 000000000..fd9a1310b --- /dev/null +++ b/src/common/utils/clair/utils.go @@ -0,0 +1,37 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clair + +import ( + "github.com/vmware/harbor/src/common/models" + "strings" +) + +// ParseClairSev parse the severity of clair to Harbor's Severity type if the string is not recognized the value will be set to unknown. +func ParseClairSev(clairSev string) models.Severity { + sev := strings.ToLower(clairSev) + switch sev { + case "negligible": + return models.SevNone + case "low": + return models.SevLow + case "medium": + return models.SevMedium + case "high": + return models.SevHigh + default: + return models.SevUnknown + } +} diff --git a/src/common/utils/clair/utils_test.go b/src/common/utils/clair/utils_test.go new file mode 100644 index 000000000..f478c38f8 --- /dev/null +++ b/src/common/utils/clair/utils_test.go @@ -0,0 +1,35 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package clair + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/common/models" +) + +func TestParseServerity(t *testing.T) { + assert := assert.New(t) + in := map[string]models.Severity{ + "negligible": models.SevNone, + "whatever": models.SevUnknown, + "LOW": models.SevLow, + "Medium": models.SevMedium, + "high": models.SevHigh, + } + for k, v := range in { + assert.Equal(v, ParseClairSev(k)) + } +} diff --git a/src/common/utils/log/logger.go b/src/common/utils/log/logger.go index 90e091407..50bd2af1f 100644 --- a/src/common/utils/log/logger.go +++ b/src/common/utils/log/logger.go @@ -64,6 +64,11 @@ func New(out io.Writer, fmtter Formatter, lvl Level) *Logger { } } +//DefaultLogger returns the default logger within the pkg, i.e. the one used in log.Infof.... +func DefaultLogger() *Logger { + return logger +} + //SetOutput sets the output of Logger l func (l *Logger) SetOutput(out io.Writer) { l.mu.Lock() diff --git a/src/common/utils/registry/auth/tokenauthorizer.go b/src/common/utils/registry/auth/tokenauthorizer.go index 2ea1f0cb0..b1766e5e0 100644 --- a/src/common/utils/registry/auth/tokenauthorizer.go +++ b/src/common/utils/registry/auth/tokenauthorizer.go @@ -25,6 +25,7 @@ import ( "time" //"github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/common/utils/registry" registry_error "github.com/vmware/harbor/src/common/utils/error" @@ -205,11 +206,7 @@ func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes [] return } - tk := struct { - Token string `json:"token"` - ExpiresIn int `json:"expires_in"` - IssuedAt string `json:"issued_at"` - }{} + tk := models.Token{} if err = json.Unmarshal(b, &tk); err != nil { return } diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index 1e2877bc6..e362d2451 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -167,3 +167,8 @@ func ExtEndpoint() (string, error) { func InternalTokenServiceEndpoint() string { return "http://ui/service/token" } + +// ClairEndpoint returns the end point of clair instance, by default it's the one deployed within Harbor. +func ClairEndpoint() string { + return "http://clair:6060" +} diff --git a/src/jobservice/job/job_test.go b/src/jobservice/job/job_test.go index 03cc0863a..48f7b608a 100644 --- a/src/jobservice/job/job_test.go +++ b/src/jobservice/job/job_test.go @@ -217,5 +217,11 @@ func prepareScanJobData() error { } func clearScanJobData() error { - return dao.ClearTable(models.ScanJobTable) + if err := dao.ClearTable(models.ScanJobTable); err != nil { + return err + } + if err := dao.ClearTable(models.ScanOverviewTable); err != nil { + return err + } + return nil } diff --git a/src/jobservice/job/jobs.go b/src/jobservice/job/jobs.go index 4a5eb48c6..87dc8d854 100644 --- a/src/jobservice/job/jobs.go +++ b/src/jobservice/job/jobs.go @@ -220,6 +220,10 @@ func (sj *ScanJob) Init() error { Tag: job.Tag, Digest: job.Digest, } + err = dao.SetScanJobForImg(job.Digest, sj.id) + if err != nil { + return err + } return nil } diff --git a/src/jobservice/job/statemachine.go b/src/jobservice/job/statemachine.go index 52d6a1f1f..196b732f7 100644 --- a/src/jobservice/job/statemachine.go +++ b/src/jobservice/job/statemachine.go @@ -258,11 +258,14 @@ func addImgScanTransition(sm *SM, parm *ScanJobParm) { Repository: parm.Repository, Tag: parm.Tag, Digest: parm.Digest, + JobID: sm.CurrentJob.ID(), Logger: sm.Logger, } + layerScanHandler := &scan.LayerScanHandler{Context: ctx} sm.AddTransition(models.JobRunning, scan.StateInitialize, &scan.Initializer{Context: ctx}) - sm.AddTransition(scan.StateInitialize, scan.StateScanLayer, &scan.LayerScanHandler{Context: ctx}) + sm.AddTransition(scan.StateInitialize, scan.StateScanLayer, layerScanHandler) + sm.AddTransition(scan.StateScanLayer, scan.StateScanLayer, layerScanHandler) sm.AddTransition(scan.StateScanLayer, scan.StateSummarize, &scan.SummarizeHandler{Context: ctx}) sm.AddTransition(scan.StateSummarize, models.JobFinished, &StatusUpdater{sm.CurrentJob, models.JobFinished}) } diff --git a/src/jobservice/scan/context.go b/src/jobservice/scan/context.go index bbdb1097d..b14fb39c7 100644 --- a/src/jobservice/scan/context.go +++ b/src/jobservice/scan/context.go @@ -15,6 +15,8 @@ package scan import ( + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/clair" "github.com/vmware/harbor/src/common/utils/log" ) @@ -29,16 +31,15 @@ const ( //JobContext is for sharing data across handlers in a execution of a scan job. type JobContext struct { + JobID int64 Repository string Tag string Digest string - //the digests of layers - layers []string - //each layer name has to be unique, so it should be ${img-digest}-${layer-digest} - layerNames []string - //the index of current layer + //The array of data object to set as request body for layer scan. + layers []models.ClairLayer current int //token for accessing the registry - token string - Logger *log.Logger + token string + clairClient *clair.Client + Logger *log.Logger } diff --git a/src/jobservice/scan/handlers.go b/src/jobservice/scan/handlers.go index e27296f6d..1d4edec82 100644 --- a/src/jobservice/scan/handlers.go +++ b/src/jobservice/scan/handlers.go @@ -15,7 +15,17 @@ package scan import ( + "github.com/docker/distribution" + "github.com/docker/distribution/manifest/schema2" + "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/clair" + "github.com/vmware/harbor/src/common/utils/registry/auth" + "github.com/vmware/harbor/src/jobservice/config" + "github.com/vmware/harbor/src/jobservice/utils" + + "fmt" + "net/http" ) // Initializer will handle the initialise state pull the manifest, prepare token. @@ -27,9 +37,60 @@ type Initializer struct { func (iz *Initializer) Enter() (string, error) { logger := iz.Context.Logger logger.Infof("Entered scan initializer") + regURL, err := config.LocalRegURL() + if err != nil { + logger.Errorf("Failed to read regURL, error: %v", err) + return "", err + } + c := &http.Cookie{Name: models.UISecretCookie, Value: config.JobserviceSecret()} + repoClient, err := utils.NewRepositoryClient(regURL, false, auth.NewCookieCredential(c), + config.InternalTokenServiceEndpoint(), iz.Context.Repository, "pull") + if err != nil { + logger.Errorf("An error occurred while creating repository client: %v", err) + return "", err + } + + _, _, payload, err := repoClient.PullManifest(iz.Context.Digest, []string{schema2.MediaTypeManifest}) + if err != nil { + logger.Errorf("Error pulling manifest for image %s:%s :%v", iz.Context.Repository, iz.Context.Tag, err) + return "", err + } + manifest, _, err := distribution.UnmarshalManifest(schema2.MediaTypeManifest, payload) + if err != nil { + logger.Error("Failed to unMarshal manifest from response") + return "", err + } + + tk, err := utils.GetTokenForRepo(iz.Context.Repository) + if err != nil { + return "", err + } + iz.Context.token = tk + iz.Context.clairClient = clair.NewClient(config.ClairEndpoint(), logger) + iz.prepareLayers(regURL, manifest.References()) return StateScanLayer, nil } +func (iz *Initializer) prepareLayers(registryEndpoint string, descriptors []distribution.Descriptor) { + // logger := iz.Context.Logger + tokenHeader := map[string]string{"Authorization": fmt.Sprintf("Bearer %s", iz.Context.token)} + for _, d := range descriptors { + if d.MediaType == schema2.MediaTypeConfig { + continue + } + l := models.ClairLayer{ + Name: fmt.Sprintf("%d-%s", iz.Context.JobID, d.Digest), + Headers: tokenHeader, + Format: "Docker", + Path: utils.BuildBlobURL(registryEndpoint, iz.Context.Repository, string(d.Digest)), + } + if len(iz.Context.layers) > 0 { + l.ParentName = iz.Context.layers[len(iz.Context.layers)-1].Name + } + iz.Context.layers = append(iz.Context.layers, l) + } +} + // Exit ... func (iz *Initializer) Exit() error { return nil @@ -43,8 +104,19 @@ type LayerScanHandler struct { // Enter ... func (ls *LayerScanHandler) Enter() (string, error) { logger := ls.Context.Logger - logger.Infof("Entered scan layer handler") - return StateSummarize, nil + currentLayer := ls.Context.layers[ls.Context.current] + logger.Infof("Entered scan layer handler, current: %d, layer name: %s", ls.Context.current, currentLayer.Name) + err := ls.Context.clairClient.ScanLayer(currentLayer) + if err != nil { + logger.Errorf("Unexpected error: %v", err) + return "", err + } + ls.Context.current++ + if ls.Context.current == len(ls.Context.layers) { + return StateSummarize, nil + } + logger.Infof("After scanning, return with next state: %s", StateScanLayer) + return StateScanLayer, nil } // Exit ... @@ -61,6 +133,46 @@ type SummarizeHandler struct { func (sh *SummarizeHandler) Enter() (string, error) { logger := sh.Context.Logger logger.Infof("Entered summarize handler") + layerName := sh.Context.layers[len(sh.Context.layers)-1].Name + logger.Infof("Top layer's name: %s, will use it to get the vulnerability result of image", layerName) + res, err := sh.Context.clairClient.GetResult(layerName) + if err != nil { + logger.Errorf("Failed to get result from Clair, error: %v", err) + return "", err + } + vulnMap := make(map[models.Severity]int) + features := res.Layer.Features + totalComponents := len(features) + logger.Infof("total features: %d", totalComponents) + var temp models.Severity + for _, f := range features { + sev := models.SevNone + for _, v := range f.Vulnerabilities { + temp = clair.ParseClairSev(v.Severity) + if temp > sev { + sev = temp + } + } + logger.Infof("Feature: %s, Severity: %d", f.Name, sev) + vulnMap[sev]++ + } + overallSev := models.SevNone + compSummary := []*models.ComponentsOverviewEntry{} + for k, v := range vulnMap { + if k > overallSev { + overallSev = k + } + entry := &models.ComponentsOverviewEntry{ + Sev: int(k), + Count: v, + } + compSummary = append(compSummary, entry) + } + compOverview := &models.ComponentsOverview{ + Total: totalComponents, + Summary: compSummary, + } + err = dao.UpdateImgScanOverview(sh.Context.Digest, layerName, overallSev, compOverview) return models.JobFinished, nil } diff --git a/src/jobservice/utils/utils.go b/src/jobservice/utils/utils.go index 1702d98c4..c39a0391d 100644 --- a/src/jobservice/utils/utils.go +++ b/src/jobservice/utils/utils.go @@ -15,9 +15,16 @@ package utils import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + + "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/registry" "github.com/vmware/harbor/src/common/utils/registry/auth" - "net/http" + "github.com/vmware/harbor/src/jobservice/config" ) //NewRepositoryClient create a repository client with scope type "reopsitory" and scope as the repository it would access. @@ -51,3 +58,44 @@ func (u *userAgentModifier) Modify(req *http.Request) error { req.Header.Set(http.CanonicalHeaderKey("User-Agent"), u.userAgent) return nil } + +// BuildBlobURL ... +func BuildBlobURL(endpoint, repository, digest string) string { + return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repository, digest) +} + +//GetTokenForRepo is a temp solution for job handler to get a token for clair. +func GetTokenForRepo(repository string) (string, error) { + u, err := url.Parse(config.InternalTokenServiceEndpoint()) + if err != nil { + return "", err + } + q := u.Query() + q.Add("service", "harbor-registry") + q.Add("scope", fmt.Sprintf("repository:%s:pull", repository)) + u.RawQuery = q.Encode() + r, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return "", err + } + c := &http.Cookie{Name: models.UISecretCookie, Value: config.JobserviceSecret()} + r.AddCookie(c) + client := &http.Client{} + resp, err := client.Do(r) + if err != nil { + return "", err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("Unexpected response from token service, code: %d, %s", resp.StatusCode, string(b)) + } + tk := models.Token{} + if err := json.Unmarshal(b, &tk); err != nil { + return "", err + } + return tk.Token, nil +} diff --git a/src/ui/service/token/authutils.go b/src/ui/service/token/authutils.go index f92c7ea34..4544818f6 100644 --- a/src/ui/service/token/authutils.go +++ b/src/ui/service/token/authutils.go @@ -23,12 +23,12 @@ import ( "strings" "time" + "github.com/docker/distribution/registry/auth/token" + "github.com/docker/libtrust" + "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/security" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/config" - - "github.com/docker/distribution/registry/auth/token" - "github.com/docker/libtrust" ) const ( @@ -132,18 +132,16 @@ func MakeRawToken(username, service string, access []*token.ResourceActions) (to return rs, expiresIn, issuedAt, nil } -type tokenJSON struct { - Token string `json:"token"` - ExpiresIn int `json:"expires_in"` - IssuedAt string `json:"issued_at"` -} - -func makeToken(username, service string, access []*token.ResourceActions) (*tokenJSON, error) { +func makeToken(username, service string, access []*token.ResourceActions) (*models.Token, error) { raw, expires, issued, err := MakeRawToken(username, service, access) if err != nil { return nil, err } - return &tokenJSON{raw, expires, issued.Format(time.RFC3339)}, nil + return &models.Token{ + Token: raw, + ExpiresIn: expires, + IssuedAt: issued.Format(time.RFC3339), + }, nil } func permToActions(p string) []string { diff --git a/src/ui/service/token/creator.go b/src/ui/service/token/creator.go index 64cc59784..359398bb9 100644 --- a/src/ui/service/token/creator.go +++ b/src/ui/service/token/creator.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/docker/distribution/registry/auth/token" + "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/security" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/config" @@ -70,7 +71,7 @@ func InitCreators() { // Creator creates a token ready to be served based on the http request. type Creator interface { - Create(r *http.Request) (*tokenJSON, error) + Create(r *http.Request) (*models.Token, error) } type imageParser interface { @@ -178,7 +179,7 @@ func (e *unauthorizedError) Error() string { return "Unauthorized" } -func (g generalCreator) Create(r *http.Request) (*tokenJSON, error) { +func (g generalCreator) Create(r *http.Request) (*models.Token, error) { var err error scopes := parseScopes(r.URL) log.Debugf("scopes: %v", scopes) diff --git a/src/ui/service/token/token_test.go b/src/ui/service/token/token_test.go index bcb933996..a6a6207bc 100644 --- a/src/ui/service/token/token_test.go +++ b/src/ui/service/token/token_test.go @@ -171,7 +171,7 @@ func TestBasicParser(t *testing.T) { for _, rec := range testList { r, err := p.parse(rec.input) if rec.expectError { - assert.Error(t, err, "Expected error for input: %s", rec.input) + assert.Error(t, err, fmt.Sprintf("Expected error for input: %s", rec.input)) } else { assert.Nil(t, err, "Expected no error for input: %s", rec.input) assert.Equal(t, rec.expect, *r, "result mismatch for input: %s", rec.input) @@ -193,7 +193,7 @@ func TestEndpointParser(t *testing.T) { for _, rec := range testList { r, err := p.parse(rec.input) if rec.expectError { - assert.Error(t, err, "Expected error for input: %s", rec.input) + assert.Error(t, err, fmt.Sprintf("Expected error for input: %s", rec.input)) } else { assert.Nil(t, err, "Expected no error for input: %s", rec.input) assert.Equal(t, rec.expect, *r, "result mismatch for input: %s", rec.input) diff --git a/src/vendor/github.com/astaxie/beego/orm/orm.go b/src/vendor/github.com/astaxie/beego/orm/orm.go index 0ffb6b869..e389a9930 100644 --- a/src/vendor/github.com/astaxie/beego/orm/orm.go +++ b/src/vendor/github.com/astaxie/beego/orm/orm.go @@ -137,10 +137,11 @@ func (o *orm) ReadOrCreate(md interface{}, col1 string, cols ...string) (bool, i if err == ErrNoRows { // Create id, err := o.Insert(md) + fmt.Printf("id when create: %d", id) return (err == nil), id, err } - return false, ind.FieldByIndex(mi.fields.pk.fieldIndex).Int(), err + return false, 0, err } // insert model data to database