From 58c499397465ae7738c1043bc788387f4354cecb Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Fri, 9 Jun 2017 14:28:51 +0800 Subject: [PATCH] add handlers in statemachine --- src/common/models/clair.go | 59 +++++++++++++++++++++++++ src/jobservice/api/scan.go | 6 ++- src/jobservice/job/job_test.go | 2 +- src/jobservice/job/jobs.go | 12 ++--- src/jobservice/job/statemachine.go | 22 +++++++++- src/jobservice/scan/context.go | 44 +++++++++++++++++++ src/jobservice/scan/handlers.go | 70 ++++++++++++++++++++++++++++++ 7 files changed, 206 insertions(+), 9 deletions(-) create mode 100644 src/common/models/clair.go create mode 100644 src/jobservice/scan/context.go create mode 100644 src/jobservice/scan/handlers.go diff --git a/src/common/models/clair.go b/src/common/models/clair.go new file mode 100644 index 000000000..13f123ad5 --- /dev/null +++ b/src/common/models/clair.go @@ -0,0 +1,59 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +//ClairLayer ... +type ClairLayer struct { + Name string `json:"Name,omitempty"` + NamespaceNames []string `json:"NamespaceNames,omitempty"` + Path string `json:"Path,omitempty"` + Headers map[string]string `json:"Headers,omitempty"` + ParentName string `json:"ParentName,omitempty"` + Format string `json:"Format,omitempty"` + Features []ClairFeature `json:"Features,omitempty"` +} + +//ClairFeature ... +type ClairFeature struct { + Name string `json:"Name,omitempty"` + NamespaceName string `json:"NamespaceName,omitempty"` + VersionFormat string `json:"VersionFormat,omitempty"` + Version string `json:"Version,omitempty"` + Vulnerabilities []ClairVulnerability `json:"Vulnerabilities,omitempty"` + AddedBy string `json:"AddedBy,omitempty"` +} + +//ClairVulnerability ... +type ClairVulnerability struct { + Name string `json:"Name,omitempty"` + NamespaceName string `json:"NamespaceName,omitempty"` + Description string `json:"Description,omitempty"` + Link string `json:"Link,omitempty"` + Severity string `json:"Severity,omitempty"` + Metadata map[string]interface{} `json:"Metadata,omitempty"` + FixedBy string `json:"FixedBy,omitempty"` + FixedIn []ClairFeature `json:"FixedIn,omitempty"` +} + +//ClairError ... +type ClairError struct { + Message string `json:"Message,omitempty"` +} + +//ClairLayerEnvelope ... +type ClairLayerEnvelope struct { + Layer *ClairLayer `json:"Layer,omitempty"` + Error *ClairError `json:"Error,omitempty"` +} diff --git a/src/jobservice/api/scan.go b/src/jobservice/api/scan.go index 123c2b763..5ead3f7de 100644 --- a/src/jobservice/api/scan.go +++ b/src/jobservice/api/scan.go @@ -22,6 +22,7 @@ import ( "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/common/utils/registry/auth" "github.com/vmware/harbor/src/jobservice/config" + "github.com/vmware/harbor/src/jobservice/job" "github.com/vmware/harbor/src/jobservice/utils" ) @@ -83,5 +84,8 @@ func (isj *ImageScanJob) Post() { isj.RenderError(http.StatusInternalServerError, "Failed to insert scan job data.") return } - log.Debugf("job id: %d", jid) + log.Debugf("Scan job id: %d", jid) + sj := job.NewScanJob(jid) + log.Debugf("Sent job to scheduler, job: %v", sj) + job.Schedule(sj) } diff --git a/src/jobservice/job/job_test.go b/src/jobservice/job/job_test.go index 8a4321ce0..03cc0863a 100644 --- a/src/jobservice/job/job_test.go +++ b/src/jobservice/job/job_test.go @@ -124,7 +124,7 @@ func TestScanJob(t *testing.T) { assert.Nil(err) j, err := dao.GetScanJob(scanJobID) assert.Equal(models.JobRetrying, j.Status) - assert.Equal("sha256:0204dc6e09fa57ab99ac40e415eb637d62c8b2571ecbbc9ca0eb5e2ad2b5c56f", sj.parm.digest) + assert.Equal("sha256:0204dc6e09fa57ab99ac40e415eb637d62c8b2571ecbbc9ca0eb5e2ad2b5c56f", sj.parm.Digest) sj2 := NewScanJob(99999) err = sj2.Init() assert.NotNil(err) diff --git a/src/jobservice/job/jobs.go b/src/jobservice/job/jobs.go index 4ecd0dd76..4a5eb48c6 100644 --- a/src/jobservice/job/jobs.go +++ b/src/jobservice/job/jobs.go @@ -176,9 +176,9 @@ type ScanJob struct { //ScanJobParm wraps the parms of a image scan job. type ScanJobParm struct { - repository string - tag string - digest string + Repository string + Tag string + Digest string } //ID returns the id of the scan @@ -216,9 +216,9 @@ func (sj *ScanJob) Init() error { return fmt.Errorf("The job doesn't exist in DB, job id: %d", sj.id) } sj.parm = &ScanJobParm{ - repository: job.Repository, - tag: job.Tag, - digest: job.Digest, + Repository: job.Repository, + Tag: job.Tag, + Digest: job.Digest, } return nil } diff --git a/src/jobservice/job/statemachine.go b/src/jobservice/job/statemachine.go index faeb14b96..52d6a1f1f 100644 --- a/src/jobservice/job/statemachine.go +++ b/src/jobservice/job/statemachine.go @@ -22,6 +22,7 @@ import ( "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/jobservice/config" "github.com/vmware/harbor/src/jobservice/replication" + "github.com/vmware/harbor/src/jobservice/scan" ) // SM is the state machine to handle job, it handles one job at a time. @@ -231,7 +232,12 @@ func (sm *SM) initTransitions() error { return fmt.Errorf("unsupported operation: %s", jobParm.Operation) } case ScanType: - log.Debugf("TODO for scan job, job: %v", sm.CurrentJob) + scanJob, ok := sm.CurrentJob.(*ScanJob) + if !ok { + //Shouldn't be here. + return fmt.Errorf("The job: %v is not a type of ScanJob", sm.CurrentJob) + } + addImgScanTransition(sm, scanJob.parm) return nil default: return fmt.Errorf("Unsupported job type: %v", sm.CurrentJob.Type()) @@ -247,6 +253,20 @@ func addTestTransition(sm *SM) error { } */ +func addImgScanTransition(sm *SM, parm *ScanJobParm) { + ctx := &scan.JobContext{ + Repository: parm.Repository, + Tag: parm.Tag, + Digest: parm.Digest, + Logger: sm.Logger, + } + + sm.AddTransition(models.JobRunning, scan.StateInitialize, &scan.Initializer{Context: ctx}) + sm.AddTransition(scan.StateInitialize, scan.StateScanLayer, &scan.LayerScanHandler{Context: ctx}) + sm.AddTransition(scan.StateScanLayer, scan.StateSummarize, &scan.SummarizeHandler{Context: ctx}) + sm.AddTransition(scan.StateSummarize, models.JobFinished, &StatusUpdater{sm.CurrentJob, models.JobFinished}) +} + func addImgTransferTransition(sm *SM, parm *RepJobParm) { base := replication.InitBaseHandler(parm.Repository, parm.LocalRegURL, config.JobserviceSecret(), parm.TargetURL, parm.TargetUsername, parm.TargetPassword, diff --git a/src/jobservice/scan/context.go b/src/jobservice/scan/context.go new file mode 100644 index 000000000..bbdb1097d --- /dev/null +++ b/src/jobservice/scan/context.go @@ -0,0 +1,44 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import ( + "github.com/vmware/harbor/src/common/utils/log" +) + +const ( + // StateInitialize in this state the handler will initialize the job context. + StateInitialize = "initialize" + // StateScanLayer in this state the handler will POST layer of clair to scan layer by layer of the image. + StateScanLayer = "scanlayer" + // StateSummarize in this state, the layers are scanned by clair it will call clair API to update vulnerability overview in Harbor DB. After this state, the job is finished. + StateSummarize = "summarize" +) + +//JobContext is for sharing data across handlers in a execution of a scan job. +type JobContext struct { + Repository string + Tag string + Digest string + //the digests of layers + layers []string + //each layer name has to be unique, so it should be ${img-digest}-${layer-digest} + layerNames []string + //the index of current layer + current int + //token for accessing the registry + token string + Logger *log.Logger +} diff --git a/src/jobservice/scan/handlers.go b/src/jobservice/scan/handlers.go new file mode 100644 index 000000000..e27296f6d --- /dev/null +++ b/src/jobservice/scan/handlers.go @@ -0,0 +1,70 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import ( + "github.com/vmware/harbor/src/common/models" +) + +// Initializer will handle the initialise state pull the manifest, prepare token. +type Initializer struct { + Context *JobContext +} + +// Enter ... +func (iz *Initializer) Enter() (string, error) { + logger := iz.Context.Logger + logger.Infof("Entered scan initializer") + return StateScanLayer, nil +} + +// Exit ... +func (iz *Initializer) Exit() error { + return nil +} + +//LayerScanHandler will call clair API to trigger scanning. +type LayerScanHandler struct { + Context *JobContext +} + +// Enter ... +func (ls *LayerScanHandler) Enter() (string, error) { + logger := ls.Context.Logger + logger.Infof("Entered scan layer handler") + return StateSummarize, nil +} + +// Exit ... +func (ls *LayerScanHandler) Exit() error { + return nil +} + +// SummarizeHandler will summarize the vulnerability and feature information of Clair, and store into Harbor's DB. +type SummarizeHandler struct { + Context *JobContext +} + +// Enter ... +func (sh *SummarizeHandler) Enter() (string, error) { + logger := sh.Context.Logger + logger.Infof("Entered summarize handler") + return models.JobFinished, nil +} + +// Exit ... +func (sh *SummarizeHandler) Exit() error { + return nil +}