Merge pull request #2481 from reasonerjt/clair-integration

add handlers in statemachine

src/common/models/clair.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+//ClairLayer ...
+type ClairLayer struct {
+	Name           string            `json:"Name,omitempty"`
+	NamespaceNames []string          `json:"NamespaceNames,omitempty"`
+	Path           string            `json:"Path,omitempty"`
+	Headers        map[string]string `json:"Headers,omitempty"`
+	ParentName     string            `json:"ParentName,omitempty"`
+	Format         string            `json:"Format,omitempty"`
+	Features       []ClairFeature    `json:"Features,omitempty"`
+}
+
+//ClairFeature ...
+type ClairFeature struct {
+	Name            string               `json:"Name,omitempty"`
+	NamespaceName   string               `json:"NamespaceName,omitempty"`
+	VersionFormat   string               `json:"VersionFormat,omitempty"`
+	Version         string               `json:"Version,omitempty"`
+	Vulnerabilities []ClairVulnerability `json:"Vulnerabilities,omitempty"`
+	AddedBy         string               `json:"AddedBy,omitempty"`
+}
+
+//ClairVulnerability ...
+type ClairVulnerability struct {
+	Name          string                 `json:"Name,omitempty"`
+	NamespaceName string                 `json:"NamespaceName,omitempty"`
+	Description   string                 `json:"Description,omitempty"`
+	Link          string                 `json:"Link,omitempty"`
+	Severity      string                 `json:"Severity,omitempty"`
+	Metadata      map[string]interface{} `json:"Metadata,omitempty"`
+	FixedBy       string                 `json:"FixedBy,omitempty"`
+	FixedIn       []ClairFeature         `json:"FixedIn,omitempty"`
+}
+
+//ClairError ...
+type ClairError struct {
+	Message string `json:"Message,omitempty"`
+}
+
+//ClairLayerEnvelope ...
+type ClairLayerEnvelope struct {
+	Layer *ClairLayer `json:"Layer,omitempty"`
+	Error *ClairError `json:"Error,omitempty"`
+}

src/jobservice/api/scan.go

@@ -22,6 +22,7 @@ import (
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/common/utils/registry/auth"
 	"github.com/vmware/harbor/src/jobservice/config"
+	"github.com/vmware/harbor/src/jobservice/job"
 	"github.com/vmware/harbor/src/jobservice/utils"
 )
 
@@ -83,5 +84,8 @@ func (isj *ImageScanJob) Post() {
 		isj.RenderError(http.StatusInternalServerError, "Failed to insert scan job data.")
 		return
 	}
-	log.Debugf("job id: %d", jid)
+	log.Debugf("Scan job id: %d", jid)
+	sj := job.NewScanJob(jid)
+	log.Debugf("Sent job to scheduler, job: %v", sj)
+	job.Schedule(sj)
 }

src/jobservice/job/job_test.go

@@ -124,7 +124,7 @@ func TestScanJob(t *testing.T) {
 	assert.Nil(err)
 	j, err := dao.GetScanJob(scanJobID)
 	assert.Equal(models.JobRetrying, j.Status)
-	assert.Equal("sha256:0204dc6e09fa57ab99ac40e415eb637d62c8b2571ecbbc9ca0eb5e2ad2b5c56f", sj.parm.digest)
+	assert.Equal("sha256:0204dc6e09fa57ab99ac40e415eb637d62c8b2571ecbbc9ca0eb5e2ad2b5c56f", sj.parm.Digest)
 	sj2 := NewScanJob(99999)
 	err = sj2.Init()
 	assert.NotNil(err)

src/jobservice/job/jobs.go

@@ -176,9 +176,9 @@ type ScanJob struct {
 
 //ScanJobParm wraps the parms of a image scan job.
 type ScanJobParm struct {
-	repository string
+	Repository string
-	tag        string
+	Tag        string
-	digest     string
+	Digest     string
 }
 
 //ID returns the id of the scan
@@ -216,9 +216,9 @@ func (sj *ScanJob) Init() error {
 		return fmt.Errorf("The job doesn't exist in DB, job id: %d", sj.id)
 	}
 	sj.parm = &ScanJobParm{
-		repository: job.Repository,
+		Repository: job.Repository,
-		tag:        job.Tag,
+		Tag:        job.Tag,
-		digest:     job.Digest,
+		Digest:     job.Digest,
 	}
 	return nil
 }

src/jobservice/job/statemachine.go

@@ -22,6 +22,7 @@ import (
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/jobservice/config"
 	"github.com/vmware/harbor/src/jobservice/replication"
+	"github.com/vmware/harbor/src/jobservice/scan"
 )
 
 // SM is the state machine to handle job, it handles one job at a time.
@@ -231,7 +232,12 @@ func (sm *SM) initTransitions() error {
 			return fmt.Errorf("unsupported operation: %s", jobParm.Operation)
 		}
 	case ScanType:
-		log.Debugf("TODO for scan job, job: %v", sm.CurrentJob)
+		scanJob, ok := sm.CurrentJob.(*ScanJob)
+		if !ok {
+			//Shouldn't be here.
+			return fmt.Errorf("The job: %v is not a type of ScanJob", sm.CurrentJob)
+		}
+		addImgScanTransition(sm, scanJob.parm)
 		return nil
 	default:
 		return fmt.Errorf("Unsupported job type: %v", sm.CurrentJob.Type())
@@ -247,6 +253,20 @@ func addTestTransition(sm *SM) error {
 }
 */
 
+func addImgScanTransition(sm *SM, parm *ScanJobParm) {
+	ctx := &scan.JobContext{
+		Repository: parm.Repository,
+		Tag:        parm.Tag,
+		Digest:     parm.Digest,
+		Logger:     sm.Logger,
+	}
+
+	sm.AddTransition(models.JobRunning, scan.StateInitialize, &scan.Initializer{Context: ctx})
+	sm.AddTransition(scan.StateInitialize, scan.StateScanLayer, &scan.LayerScanHandler{Context: ctx})
+	sm.AddTransition(scan.StateScanLayer, scan.StateSummarize, &scan.SummarizeHandler{Context: ctx})
+	sm.AddTransition(scan.StateSummarize, models.JobFinished, &StatusUpdater{sm.CurrentJob, models.JobFinished})
+}
+
 func addImgTransferTransition(sm *SM, parm *RepJobParm) {
 	base := replication.InitBaseHandler(parm.Repository, parm.LocalRegURL, config.JobserviceSecret(),
 		parm.TargetURL, parm.TargetUsername, parm.TargetPassword,

src/jobservice/scan/context.go

@@ -0,0 +1,44 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scan
+
+import (
+	"github.com/vmware/harbor/src/common/utils/log"
+)
+
+const (
+	// StateInitialize in this state the handler will initialize the job context.
+	StateInitialize = "initialize"
+	// StateScanLayer in this state the handler will POST layer  of clair to scan layer by layer of the image.
+	StateScanLayer = "scanlayer"
+	// StateSummarize in this state, the layers are scanned by clair it will call clair API to update vulnerability overview in Harbor DB. After this state, the job is finished.
+	StateSummarize = "summarize"
+)
+
+//JobContext is for sharing data across handlers in a execution of a scan job.
+type JobContext struct {
+	Repository string
+	Tag        string
+	Digest     string
+	//the digests of layers
+	layers []string
+	//each layer name has to be unique, so it should be ${img-digest}-${layer-digest}
+	layerNames []string
+	//the index of current layer
+	current int
+	//token for accessing the registry
+	token  string
+	Logger *log.Logger
+}

src/jobservice/scan/handlers.go

@@ -0,0 +1,70 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scan
+
+import (
+	"github.com/vmware/harbor/src/common/models"
+)
+
+// Initializer will handle the initialise state pull the manifest, prepare token.
+type Initializer struct {
+	Context *JobContext
+}
+
+// Enter ...
+func (iz *Initializer) Enter() (string, error) {
+	logger := iz.Context.Logger
+	logger.Infof("Entered scan initializer")
+	return StateScanLayer, nil
+}
+
+// Exit ...
+func (iz *Initializer) Exit() error {
+	return nil
+}
+
+//LayerScanHandler will call clair API to trigger scanning.
+type LayerScanHandler struct {
+	Context *JobContext
+}
+
+// Enter ...
+func (ls *LayerScanHandler) Enter() (string, error) {
+	logger := ls.Context.Logger
+	logger.Infof("Entered scan layer handler")
+	return StateSummarize, nil
+}
+
+// Exit ...
+func (ls *LayerScanHandler) Exit() error {
+	return nil
+}
+
+// SummarizeHandler will summarize the vulnerability and feature information of Clair, and store into Harbor's DB.
+type SummarizeHandler struct {
+	Context *JobContext
+}
+
+// Enter ...
+func (sh *SummarizeHandler) Enter() (string, error) {
+	logger := sh.Context.Logger
+	logger.Infof("Entered summarize handler")
+	return models.JobFinished, nil
+}
+
+// Exit ...
+func (sh *SummarizeHandler) Exit() error {
+	return nil
+}