Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2025-02-01 20:41:22 +01:00
Code Issues Projects Releases Wiki Activity

refactory for scan job service (#2459)

Browse Source 
* refactory for scan job service and implement ScanJob.
This commit is contained in:
Daniel Jiang 2017-06-08 15:04:23 +08:00 committed by GitHub
parent 2e5a4003a8
commit 42984fe1c9
11 changed files with 310 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
make/common/db/registry.sql
View File

@ -174,7 +174,7 @@ create table img_scan_job (
status varchar(64) NOT NULL,
repository varchar(256) NOT NULL,
tag varchar(128) NOT NULL,
digest varchar(64),
digest varchar(128),
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
PRIMARY KEY (id)

3
src/common/dao/dao_test.go
View File

@ -1637,7 +1637,7 @@ var sj2 = models.ScanJob{
Status: models.JobPending,
Repository: "library/ubuntu",
Tag: "15.10",
Digest: "sha256:1234567890",
Digest: "sha256:0204dc6e09fa57ab99ac40e415eb637d62c8b2571ecbbc9ca0eb5e2ad2b5c56f",
}
func TestAddScanJob(t *testing.T) {
@ -1692,5 +1692,4 @@ func TestUpdateScanJobStatus(t *testing.T) {
assert.NotNil(err)
err = ClearTable(models.ScanJobTable)
assert.Nil(err)
}

3
src/common/dao/scan_job.go
View File

@ -25,6 +25,9 @@ import (
// AddScanJob ...
func AddScanJob(job models.ScanJob) (int64, error) {
o := GetOrmer()
if len(job.Status) == 0 {
job.Status = models.JobPending
}
return o.Insert(&job)
}

43
src/jobservice/api/base.go Normal file
View File

@ -0,0 +1,43 @@
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"github.com/vmware/harbor/src/common/api"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/jobservice/config"
"net/http"
)
type jobBaseAPI struct {
api.BaseAPI
}
func (j *jobBaseAPI) authenticate() {
cookie, err := j.Ctx.Request.Cookie(models.UISecretCookie)
if err != nil && err != http.ErrNoCookie {
log.Errorf("failed to get cookie %s: %v", models.UISecretCookie, err)
j.CustomAbort(http.StatusInternalServerError, "")
}
if err == http.ErrNoCookie {
j.CustomAbort(http.StatusUnauthorized, "")
}
if cookie.Value != config.UISecret() {
j.CustomAbort(http.StatusForbidden, "")
}
}

19
src/jobservice/api/replication.go
View File

@ -21,7 +21,6 @@ import (
"net/http"
"strconv"
"github.com/vmware/harbor/src/common/api"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
u "github.com/vmware/harbor/src/common/utils"
@ -33,7 +32,7 @@ import (
// ReplicationJob handles /api/replicationJobs /api/replicationJobs/:id/log
// /api/replicationJobs/actions
type ReplicationJob struct {
api.BaseAPI
jobBaseAPI
}
// ReplicationReq holds informations of request for /api/replicationJobs
@ -49,22 +48,6 @@ func (rj *ReplicationJob) Prepare() {
rj.authenticate()
}
func (rj *ReplicationJob) authenticate() {
cookie, err := rj.Ctx.Request.Cookie(models.UISecretCookie)
if err != nil && err != http.ErrNoCookie {
log.Errorf("failed to get cookie %s: %v", models.UISecretCookie, err)
rj.CustomAbort(http.StatusInternalServerError, "")
}
if err == http.ErrNoCookie {
rj.CustomAbort(http.StatusUnauthorized, "")
}
if cookie.Value != config.UISecret() {
rj.CustomAbort(http.StatusForbidden, "")
}
}
// Post creates replication jobs according to the policy.
func (rj *ReplicationJob) Post() {
var data ReplicationReq

87
src/jobservice/api/scan.go Normal file
View File

@ -0,0 +1,87 @@
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"net/http"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/common/utils/registry/auth"
"github.com/vmware/harbor/src/jobservice/config"
"github.com/vmware/harbor/src/jobservice/utils"
)
// ImageScanJob handles /api/imageScanJobs /api/imageScanJobs/:id/log
type ImageScanJob struct {
jobBaseAPI
}
type imageScanReq struct {
Repo string `json:"repository"`
Tag string `json:"tag"`
}
// Prepare ...
func (isj *ImageScanJob) Prepare() {
//TODO:add authenticate to check secret when integrate with UI API.
//isj.authenticate()
}
// Post creates a scanner job and hand it to statemachine.
func (isj *ImageScanJob) Post() {
var data imageScanReq
isj.DecodeJSONReq(&data)
log.Debugf("data: %+v", data)
regURL, err := config.LocalRegURL()
if err != nil {
log.Errorf("Failed to read regURL, error: %v", err)
isj.RenderError(http.StatusInternalServerError, "Failed to read registry URL from config")
return
}
c := &http.Cookie{Name: models.UISecretCookie, Value: config.JobserviceSecret()}
repoClient, err := utils.NewRepositoryClient(regURL, false, auth.NewCookieCredential(c),
config.InternalTokenServiceEndpoint(), data.Repo, "pull", "push", "*")
if err != nil {
log.Errorf("An error occurred while creating repository client: %v", err)
isj.RenderError(http.StatusInternalServerError, "Failed to repository client")
return
}
digest, exist, err := repoClient.ManifestExist(data.Tag)
if err != nil {
log.Errorf("Failed to get manifest, error: %v", err)
isj.RenderError(http.StatusInternalServerError, "Failed to get manifest")
return
}
if !exist {
log.Errorf("The repository based on request: %+v does not exist", data)
isj.RenderError(http.StatusNotFound, "")
return
}
//Insert job into DB
j := models.ScanJob{
Repository: data.Repo,
Tag: data.Tag,
Digest: digest,
}
jid, err := dao.AddScanJob(j)
if err != nil {
log.Errorf("Failed to add scan job to DB, error: %v", err)
isj.RenderError(http.StatusInternalServerError, "Failed to insert scan job data.")
return
}
log.Debugf("job id: %d", jid)
}

55
src/jobservice/job/job_test.go
View File

@ -23,10 +23,11 @@ import (
"github.com/vmware/harbor/src/common/utils/test"
"github.com/vmware/harbor/src/jobservice/config"
"os"
"strconv"
"testing"
)
var repJobID int64
var repJobID, scanJobID int64
func TestMain(m *testing.M) {
//Init config...
@ -34,6 +35,13 @@ func TestMain(m *testing.M) {
if len(os.Getenv("MYSQL_HOST")) > 0 {
conf[common.MySQLHost] = os.Getenv("MYSQL_HOST")
}
if len(os.Getenv("MYSQL_PORT")) > 0 {
p, err := strconv.Atoi(os.Getenv("MYSQL_PORT"))
if err != nil {
panic(err)
}
conf[common.MySQLPort] = p
}
if len(os.Getenv("MYSQL_USR")) > 0 {
conf[common.MySQLUsername] = os.Getenv("MYSQL_USR")
}
@ -72,8 +80,12 @@ func TestMain(m *testing.M) {
if err := prepareRepJobData(); err != nil {
log.Fatalf("failed to initialised databse, error: %v", err)
}
if err := prepareScanJobData(); err != nil {
log.Fatalf("failed to initialised databse, error: %v", err)
}
rc := m.Run()
clearRepJobData()
clearScanJobData()
if rc != 0 {
os.Exit(rc)
}
@ -99,6 +111,25 @@ func TestRepJob(t *testing.T) {
assert.NotNil(err)
}
func TestScanJob(t *testing.T) {
sj := NewScanJob(scanJobID)
assert := assert.New(t)
err := sj.Init()
assert.Nil(err)
assert.Equal(scanJobID, sj.ID())
assert.Equal(ScanType, sj.Type())
p := fmt.Sprintf("/var/log/jobs/scan_job/job_%d.log", scanJobID)
assert.Equal(p, sj.LogPath())
err = sj.UpdateStatus(models.JobRetrying)
assert.Nil(err)
j, err := dao.GetScanJob(scanJobID)
assert.Equal(models.JobRetrying, j.Status)
assert.Equal("sha256:0204dc6e09fa57ab99ac40e415eb637d62c8b2571ecbbc9ca0eb5e2ad2b5c56f", sj.parm.digest)
sj2 := NewScanJob(99999)
err = sj2.Init()
assert.NotNil(err)
}
func TestStatusUpdater(t *testing.T) {
assert := assert.New(t)
rj := NewRepJob(repJobID)
@ -166,3 +197,25 @@ func clearRepJobData() error {
}
return nil
}
func prepareScanJobData() error {
if err := clearScanJobData(); err != nil {
return err
}
sj := models.ScanJob{
Status: models.JobPending,
Repository: "library/ubuntu",
Tag: "15.10",
Digest: "sha256:0204dc6e09fa57ab99ac40e415eb637d62c8b2571ecbbc9ca0eb5e2ad2b5c56f",
}
id, err := dao.AddScanJob(sj)
if err != nil {
return err
}
scanJobID = id
return nil
}
func clearScanJobData() error {
return dao.ClearTable(models.ScanJobTable)
}

61
src/jobservice/job/jobs.go
View File

@ -20,6 +20,7 @@ import (
"github.com/vmware/harbor/src/jobservice/config"
"fmt"
"path/filepath"
)
// Type is for job Type
@ -166,3 +167,63 @@ func (rj *RepJob) Init() error {
func NewRepJob(id int64) *RepJob {
return &RepJob{id: id}
}
//ScanJob implements the Job interface, representing a job for scanning image.
type ScanJob struct {
id int64
parm *ScanJobParm
}
//ScanJobParm wraps the parms of a image scan job.
type ScanJobParm struct {
repository string
tag string
digest string
}
//ID returns the id of the scan
func (sj *ScanJob) ID() int64 {
return sj.id
}
//Type always return ScanType
func (sj *ScanJob) Type() Type {
return ScanType
}
//LogPath returns the absolute path of the log file for the job, log files for scan job will be put in a sub folder of base log path.
func (sj *ScanJob) LogPath() string {
return GetJobLogPath(filepath.Join(config.LogDir(), "scan_job"), sj.id)
}
//String ...
func (sj *ScanJob) String() string {
return fmt.Sprintf("{JobID: %d, JobType: %v}", sj.ID(), sj.Type())
}
//UpdateStatus ...
func (sj *ScanJob) UpdateStatus(status string) error {
return dao.UpdateScanJobStatus(sj.id, status)
}
//Init query the DB and populate the information of the image to scan in the parm of this job.
func (sj *ScanJob) Init() error {
job, err := dao.GetScanJob(sj.id)
if err != nil {
return fmt.Errorf("Failed to get job, error: %v", err)
}
if job == nil {
return fmt.Errorf("The job doesn't exist in DB, job id: %d", sj.id)
}
sj.parm = &ScanJobParm{
repository: job.Repository,
tag: job.Tag,
digest: job.Digest,
}
return nil
}
//NewScanJob creates a instance of ScanJob by id.
func NewScanJob(id int64) *ScanJob {
return &ScanJob{id: id}
}

41
src/jobservice/replication/transfer.go
View File

@ -33,6 +33,7 @@ import (
"github.com/vmware/harbor/src/common/utils/registry"
"github.com/vmware/harbor/src/common/utils/registry/auth"
"github.com/vmware/harbor/src/jobservice/config"
"github.com/vmware/harbor/src/jobservice/utils"
)
const (
@ -136,8 +137,8 @@ func (i *Initializer) Enter() (string, error) {
func (i *Initializer) enter() (string, error) {
c := &http.Cookie{Name: models.UISecretCookie, Value: i.srcSecret}
srcCred := auth.NewCookieCredential(c)
srcClient, err := newRepositoryClient(i.srcURL, i.insecure, srcCred,
config.InternalTokenServiceEndpoint(), i.repository, "repository", i.repository, "pull", "push", "*")
srcClient, err := utils.NewRepositoryClient(i.srcURL, i.insecure, srcCred,
config.InternalTokenServiceEndpoint(), i.repository, "pull", "push", "*")
if err != nil {
i.logger.Errorf("an error occurred while creating source repository client: %v", err)
return "", err
@ -145,8 +146,8 @@ func (i *Initializer) enter() (string, error) {
i.srcClient = srcClient
dstCred := auth.NewBasicAuthCredential(i.dstUsr, i.dstPwd)
dstClient, err := newRepositoryClient(i.dstURL, i.insecure, dstCred,
"", i.repository, "repository", i.repository, "pull", "push", "*")
dstClient, err := utils.NewRepositoryClient(i.dstURL, i.insecure, dstCred,
"", i.repository, "pull", "push", "*")
if err != nil {
i.logger.Errorf("an error occurred while creating destination repository client: %v", err)
return "", err
@ -450,35 +451,3 @@ func (m *ManifestPusher) enter() (string, error) {
return StatePullManifest, nil
}
func newRepositoryClient(endpoint string, insecure bool, credential auth.Credential,
tokenServiceEndpoint, repository, scopeType, scopeName string,
scopeActions ...string) (*registry.Repository, error) {
authorizer := auth.NewStandardTokenAuthorizer(credential, insecure,
tokenServiceEndpoint, scopeType, scopeName, scopeActions...)
store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
if err != nil {
return nil, err
}
uam := &userAgentModifier{
userAgent: "harbor-registry-client",
}
client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store, uam)
if err != nil {
return nil, err
}
return client, nil
}
type userAgentModifier struct {
userAgent string
}
// Modify adds user-agent header to the request
func (u *userAgentModifier) Modify(req *http.Request) error {
req.Header.Set(http.CanonicalHeaderKey("User-Agent"), u.userAgent)
return nil
}

1
src/jobservice/router.go
View File

@ -24,4 +24,5 @@ func initRouters() {
beego.Router("/api/jobs/replication", &api.ReplicationJob{})
beego.Router("/api/jobs/replication/:id/log", &api.ReplicationJob{}, "get:GetLog")
beego.Router("/api/jobs/replication/actions", &api.ReplicationJob{}, "post:HandleAction")
beego.Router("/api/jobs/scan", &api.ImageScanJob{})
}

53
src/jobservice/utils/utils.go Normal file
View File

@ -0,0 +1,53 @@
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (
"github.com/vmware/harbor/src/common/utils/registry"
"github.com/vmware/harbor/src/common/utils/registry/auth"
"net/http"
)
//NewRepositoryClient create a repository client with scope type "reopsitory" and scope as the repository it would access.
func NewRepositoryClient(endpoint string, insecure bool, credential auth.Credential,
tokenServiceEndpoint, repository string, actions ...string) (*registry.Repository, error) {
authorizer := auth.NewStandardTokenAuthorizer(credential, insecure,
tokenServiceEndpoint, "repository", repository, actions...)
store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
if err != nil {
return nil, err
}
uam := &userAgentModifier{
userAgent: "harbor-registry-client",
}
client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store, uam)
if err != nil {
return nil, err
}
return client, nil
}
type userAgentModifier struct {
userAgent string
}
// Modify adds user-agent header to the request
func (u *userAgentModifier) Modify(req *http.Request) error {
req.Header.Set(http.CanonicalHeaderKey("User-Agent"), u.userAgent)
return nil
}