Merge pull request #2291 from reasonerjt/vulscan-job-refactory

add scan job table and dao functions

make/common/db/registry.sql

@@ -169,6 +169,17 @@ create table replication_job (
  INDEX poid_uptime (policy_id, update_time)
  );
  
+create table img_scan_job (
+ id int NOT NULL AUTO_INCREMENT,
+ status varchar(64) NOT NULL,
+ repository varchar(256) NOT NULL,
+ tag   varchar(128) NOT NULL,
+ digest varchar(64),
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
+ PRIMARY KEY (id)
+ );
+
 create table properties (
  k varchar(64) NOT NULL,
  v varchar(128) NOT NULL,

make/common/db/registry_sqlite.sql

@@ -160,6 +160,17 @@ create table replication_job (
  update_time timestamp default CURRENT_TIMESTAMP
  );
 
+
+create table img_scan_job (
+ id INTEGER PRIMARY KEY,
+ status varchar(64) NOT NULL,
+ repository varchar(256) NOT NULL,
+ tag   varchar(128) NOT NULL,
+ digest varchar(64),
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ update_time timestamp default CURRENT_TIMESTAMP
+ );
+
 CREATE INDEX policy ON replication_job (policy_id);
 CREATE INDEX poid_uptime ON replication_job (policy_id, update_time);
  

src/common/dao/dao_test.go

@@ -15,6 +15,7 @@
 package dao
 
 import (
+	"fmt"
 	"os"
 	"strconv"
 	"testing"
@@ -41,6 +42,13 @@ func execUpdate(o orm.Ormer, sql string, params ...interface{}) error {
 	return nil
 }
 
+func clearTable(table string) error {
+	o := GetOrmer()
+	sql := fmt.Sprintf("delete from %s where 1=1", table)
+	_, err := o.Raw(sql).Exec()
+	return err
+}
+
 func clearUp(username string) {
 	var err error
 
@@ -1649,3 +1657,71 @@ func TestDeleteRepository(t *testing.T) {
 		t.Errorf("repository is not nil after deletion, repository: %+v", repository)
 	}
 }
+
+var sj1 = models.ScanJob{
+	Status:     models.JobPending,
+	Repository: "library/ubuntu",
+	Tag:        "14.04",
+}
+
+var sj2 = models.ScanJob{
+	Status:     models.JobPending,
+	Repository: "library/ubuntu",
+	Tag:        "15.10",
+	Digest:     "sha256:1234567890",
+}
+
+func TestAddScanJob(t *testing.T) {
+	assert := assert.New(t)
+	id, err := AddScanJob(sj1)
+	assert.Nil(err)
+	r1, err := GetScanJob(id)
+	assert.Nil(err)
+	assert.Equal(sj1.Tag, r1.Tag)
+	assert.Equal(sj1.Status, r1.Status)
+	assert.Equal(sj1.Repository, r1.Repository)
+	err = clearTable(ScanJobTable)
+	assert.Nil(err)
+}
+
+func TestGetScanJobs(t *testing.T) {
+	assert := assert.New(t)
+	_, err := AddScanJob(sj1)
+	assert.Nil(err)
+	id2, err := AddScanJob(sj1)
+	assert.Nil(err)
+	_, err = AddScanJob(sj2)
+	assert.Nil(err)
+	r, err := GetScanJobsByImage("library/ubuntu", "14.04")
+	assert.Nil(err)
+	assert.Equal(2, len(r))
+	assert.Equal(id2, r[0].ID)
+	r, err = GetScanJobsByImage("library/ubuntu", "14.04", 1)
+	assert.Nil(err)
+	assert.Equal(1, len(r))
+	r, err = GetScanJobsByDigest("sha256:nono")
+	assert.Nil(err)
+	assert.Equal(0, len(r))
+	r, err = GetScanJobsByDigest(sj2.Digest)
+	assert.Equal(1, len(r))
+	assert.Equal(sj2.Tag, r[0].Tag)
+	assert.Nil(err)
+	err = clearTable(ScanJobTable)
+	assert.Nil(err)
+}
+
+func TestUpdateScanJobStatus(t *testing.T) {
+	assert := assert.New(t)
+	id, err := AddScanJob(sj1)
+	assert.Nil(err)
+	err = UpdateScanJobStatus(id, "newstatus")
+	assert.Nil(err)
+	j, err := GetScanJob(id)
+	assert.Nil(err)
+	assert.Equal("newstatus", j.Status)
+	err = UpdateScanJobStatus(id+9, "newstatus")
+	assert.NotNil(err)
+	err = clearTable(ScanJobTable)
+	assert.Nil(err)
+
+}

src/common/dao/scan_job.go

@@ -0,0 +1,81 @@
+// copyright (c) 2017 vmware, inc. all rights reserved.
+//
+// licensed under the apache license, version 2.0 (the "license");
+// you may not use this file except in compliance with the license.
+// you may obtain a copy of the license at
+//
+//    http://www.apache.org/licenses/license-2.0
+//
+// unless required by applicable law or agreed to in writing, software
+// distributed under the license is distributed on an "as is" basis,
+// without warranties or conditions of any kind, either express or implied.
+// see the license for the specific language governing permissions and
+// limitations under the license.
+
+package dao
+
+import (
+	"github.com/astaxie/beego/orm"
+	"github.com/vmware/harbor/src/common/models"
+
+	"fmt"
+	"time"
+)
+
+// ScanJobTable is the table name of scan jobs.
+const ScanJobTable = "img_scan_job"
+
+// AddScanJob ...
+func AddScanJob(job models.ScanJob) (int64, error) {
+	o := GetOrmer()
+	return o.Insert(&job)
+}
+
+// GetScanJob ...
+func GetScanJob(id int64) (*models.ScanJob, error) {
+	o := GetOrmer()
+	j := models.ScanJob{ID: id}
+	err := o.Read(&j)
+	if err == orm.ErrNoRows {
+		return nil, nil
+	}
+	return &j, nil
+}
+
+// GetScanJobsByImage returns a list of scan jobs with given repository and tag
+func GetScanJobsByImage(repository, tag string, limit ...int) ([]*models.ScanJob, error) {
+	var res []*models.ScanJob
+	_, err := scanJobQs(limit...).Filter("repository", repository).Filter("tag", tag).OrderBy("-id").All(&res)
+	return res, err
+}
+
+// GetScanJobsByDigest returns a list of scan jobs with given digest
+func GetScanJobsByDigest(digest string, limit ...int) ([]*models.ScanJob, error) {
+	var res []*models.ScanJob
+	_, err := scanJobQs(limit...).Filter("digest", digest).OrderBy("-id").All(&res)
+	return res, err
+}
+
+// UpdateScanJobStatus updates the status of a scan job.
+func UpdateScanJobStatus(id int64, status string) error {
+	o := GetOrmer()
+	sj := models.ScanJob{
+		ID:         id,
+		Status:     status,
+		UpdateTime: time.Now(),
+	}
+	n, err := o.Update(&sj, "Status", "UpdateTime")
+	if n == 0 {
+		return fmt.Errorf("Failed to update scan job with id: %d, error: %v", id, err)
+	}
+	return err
+}
+
+func scanJobQs(limit ...int) orm.QuerySeter {
+	o := GetOrmer()
+	l := -1
+	if len(limit) == 1 {
+		l = limit[0]
+	}
+	return o.QueryTable(ScanJobTable).Limit(l)
+}

src/common/models/base.go

@@ -26,5 +26,6 @@ func init() {
 		new(Project),
 		new(Role),
 		new(AccessLog),
+		new(ScanJob),
 		new(RepoRecord))
 }

src/common/models/job.go

@@ -0,0 +1,34 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+const (
+	//JobPending ...
+	JobPending string = "pending"
+	//JobRunning ...
+	JobRunning string = "running"
+	//JobError ...
+	JobError string = "error"
+	//JobStopped ...
+	JobStopped string = "stopped"
+	//JobFinished ...
+	JobFinished string = "finished"
+	//JobCanceled ...
+	JobCanceled string = "canceled"
+	//JobRetrying indicate the job needs to be retried, it will be scheduled to the end of job queue by statemachine after an interval.
+	JobRetrying string = "retrying"
+	//JobContinue is the status returned by statehandler to tell statemachine to move to next possible state based on trasition table.
+	JobContinue string = "_continue"
+)

src/common/models/replication_job.go

@@ -22,22 +22,6 @@ import (
 )
 
 const (
-	//JobPending ...
-	JobPending string = "pending"
-	//JobRunning ...
-	JobRunning string = "running"
-	//JobError ...
-	JobError string = "error"
-	//JobStopped ...
-	JobStopped string = "stopped"
-	//JobFinished ...
-	JobFinished string = "finished"
-	//JobCanceled ...
-	JobCanceled string = "canceled"
-	//JobRetrying indicate the job needs to be retried, it will be scheduled to the end of job queue by statemachine after an interval.
-	JobRetrying string = "retrying"
-	//JobContinue is the status returned by statehandler to tell statemachine to move to next possible state based on trasition table.
-	JobContinue string = "_continue"
 	//RepOpTransfer represents the operation of a job to transfer repository to a remote registry/harbor instance.
 	RepOpTransfer string = "transfer"
 	//RepOpDelete represents the operation of a job to remove repository from a remote registry/harbor instance.

src/common/models/scan_job.go

@@ -0,0 +1,33 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import "time"
+
+//ScanJob is the model to represent a job for image scan in DB.
+type ScanJob struct {
+	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
+	Status       string    `orm:"column(status)" json:"status"`
+	Repository   string    `orm:"column(repository)" json:"repository"`
+	Tag          string    `orm:"column(tag)" json:"tag"`
+	Digest       string    `orm:"column(digest)" json:"digest"`
+	CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
+	UpdateTime   time.Time `orm:"column(update_time);auto_now" json:"update_time"`
+}
+
+//TableName is required by by beego orm to map ScanJob to table img_scan_job
+func (s *ScanJob) TableName() string {
+	return "img_scan_job"
+}