diff --git a/make/common/db/registry.sql b/make/common/db/registry.sql index 185528e5a..62d301b6d 100644 --- a/make/common/db/registry.sql +++ b/make/common/db/registry.sql @@ -174,7 +174,7 @@ create table img_scan_job ( status varchar(64) NOT NULL, repository varchar(256) NOT NULL, tag varchar(128) NOT NULL, - digest varchar(64), + digest varchar(128), creation_time timestamp default CURRENT_TIMESTAMP, update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, PRIMARY KEY (id) diff --git a/src/common/dao/dao_test.go b/src/common/dao/dao_test.go index 039cda9df..7114f6a98 100644 --- a/src/common/dao/dao_test.go +++ b/src/common/dao/dao_test.go @@ -1637,7 +1637,7 @@ var sj2 = models.ScanJob{ Status: models.JobPending, Repository: "library/ubuntu", Tag: "15.10", - Digest: "sha256:1234567890", + Digest: "sha256:0204dc6e09fa57ab99ac40e415eb637d62c8b2571ecbbc9ca0eb5e2ad2b5c56f", } func TestAddScanJob(t *testing.T) { @@ -1692,5 +1692,4 @@ func TestUpdateScanJobStatus(t *testing.T) { assert.NotNil(err) err = ClearTable(models.ScanJobTable) assert.Nil(err) - } diff --git a/src/common/dao/scan_job.go b/src/common/dao/scan_job.go index 9e2d06bee..945c5af17 100644 --- a/src/common/dao/scan_job.go +++ b/src/common/dao/scan_job.go @@ -25,6 +25,9 @@ import ( // AddScanJob ... func AddScanJob(job models.ScanJob) (int64, error) { o := GetOrmer() + if len(job.Status) == 0 { + job.Status = models.JobPending + } return o.Insert(&job) } diff --git a/src/jobservice/api/base.go b/src/jobservice/api/base.go new file mode 100644 index 000000000..59bbb93b5 --- /dev/null +++ b/src/jobservice/api/base.go @@ -0,0 +1,43 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "github.com/vmware/harbor/src/common/api" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" + "net/http" +) + +type jobBaseAPI struct { + api.BaseAPI +} + +func (j *jobBaseAPI) authenticate() { + cookie, err := j.Ctx.Request.Cookie(models.UISecretCookie) + if err != nil && err != http.ErrNoCookie { + log.Errorf("failed to get cookie %s: %v", models.UISecretCookie, err) + j.CustomAbort(http.StatusInternalServerError, "") + } + + if err == http.ErrNoCookie { + j.CustomAbort(http.StatusUnauthorized, "") + } + + if cookie.Value != config.UISecret() { + j.CustomAbort(http.StatusForbidden, "") + } +} diff --git a/src/jobservice/api/replication.go b/src/jobservice/api/replication.go index 169e54f5f..f9d579560 100644 --- a/src/jobservice/api/replication.go +++ b/src/jobservice/api/replication.go @@ -21,7 +21,6 @@ import ( "net/http" "strconv" - "github.com/vmware/harbor/src/common/api" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" u "github.com/vmware/harbor/src/common/utils" @@ -33,7 +32,7 @@ import ( // ReplicationJob handles /api/replicationJobs /api/replicationJobs/:id/log // /api/replicationJobs/actions type ReplicationJob struct { - api.BaseAPI + jobBaseAPI } // ReplicationReq holds informations of request for /api/replicationJobs @@ -49,22 +48,6 @@ func (rj *ReplicationJob) Prepare() { rj.authenticate() } -func (rj *ReplicationJob) authenticate() { - cookie, err := rj.Ctx.Request.Cookie(models.UISecretCookie) - if err != nil && err != http.ErrNoCookie { - log.Errorf("failed to get cookie %s: %v", models.UISecretCookie, err) - rj.CustomAbort(http.StatusInternalServerError, "") - } - - if err == http.ErrNoCookie { - rj.CustomAbort(http.StatusUnauthorized, "") - } - - if cookie.Value != config.UISecret() { - rj.CustomAbort(http.StatusForbidden, "") - } -} - // Post creates replication jobs according to the policy. func (rj *ReplicationJob) Post() { var data ReplicationReq diff --git a/src/jobservice/api/scan.go b/src/jobservice/api/scan.go new file mode 100644 index 000000000..123c2b763 --- /dev/null +++ b/src/jobservice/api/scan.go @@ -0,0 +1,87 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "net/http" + + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/common/utils/registry/auth" + "github.com/vmware/harbor/src/jobservice/config" + "github.com/vmware/harbor/src/jobservice/utils" +) + +// ImageScanJob handles /api/imageScanJobs /api/imageScanJobs/:id/log +type ImageScanJob struct { + jobBaseAPI +} + +type imageScanReq struct { + Repo string `json:"repository"` + Tag string `json:"tag"` +} + +// Prepare ... +func (isj *ImageScanJob) Prepare() { + //TODO:add authenticate to check secret when integrate with UI API. + //isj.authenticate() +} + +// Post creates a scanner job and hand it to statemachine. +func (isj *ImageScanJob) Post() { + var data imageScanReq + isj.DecodeJSONReq(&data) + log.Debugf("data: %+v", data) + regURL, err := config.LocalRegURL() + if err != nil { + log.Errorf("Failed to read regURL, error: %v", err) + isj.RenderError(http.StatusInternalServerError, "Failed to read registry URL from config") + return + } + c := &http.Cookie{Name: models.UISecretCookie, Value: config.JobserviceSecret()} + repoClient, err := utils.NewRepositoryClient(regURL, false, auth.NewCookieCredential(c), + config.InternalTokenServiceEndpoint(), data.Repo, "pull", "push", "*") + if err != nil { + log.Errorf("An error occurred while creating repository client: %v", err) + isj.RenderError(http.StatusInternalServerError, "Failed to repository client") + return + } + digest, exist, err := repoClient.ManifestExist(data.Tag) + if err != nil { + log.Errorf("Failed to get manifest, error: %v", err) + isj.RenderError(http.StatusInternalServerError, "Failed to get manifest") + return + } + if !exist { + log.Errorf("The repository based on request: %+v does not exist", data) + isj.RenderError(http.StatusNotFound, "") + return + } + //Insert job into DB + j := models.ScanJob{ + Repository: data.Repo, + Tag: data.Tag, + Digest: digest, + } + jid, err := dao.AddScanJob(j) + if err != nil { + log.Errorf("Failed to add scan job to DB, error: %v", err) + isj.RenderError(http.StatusInternalServerError, "Failed to insert scan job data.") + return + } + log.Debugf("job id: %d", jid) +} diff --git a/src/jobservice/job/job_test.go b/src/jobservice/job/job_test.go index 7a8126489..8a4321ce0 100644 --- a/src/jobservice/job/job_test.go +++ b/src/jobservice/job/job_test.go @@ -23,10 +23,11 @@ import ( "github.com/vmware/harbor/src/common/utils/test" "github.com/vmware/harbor/src/jobservice/config" "os" + "strconv" "testing" ) -var repJobID int64 +var repJobID, scanJobID int64 func TestMain(m *testing.M) { //Init config... @@ -34,6 +35,13 @@ func TestMain(m *testing.M) { if len(os.Getenv("MYSQL_HOST")) > 0 { conf[common.MySQLHost] = os.Getenv("MYSQL_HOST") } + if len(os.Getenv("MYSQL_PORT")) > 0 { + p, err := strconv.Atoi(os.Getenv("MYSQL_PORT")) + if err != nil { + panic(err) + } + conf[common.MySQLPort] = p + } if len(os.Getenv("MYSQL_USR")) > 0 { conf[common.MySQLUsername] = os.Getenv("MYSQL_USR") } @@ -72,8 +80,12 @@ func TestMain(m *testing.M) { if err := prepareRepJobData(); err != nil { log.Fatalf("failed to initialised databse, error: %v", err) } + if err := prepareScanJobData(); err != nil { + log.Fatalf("failed to initialised databse, error: %v", err) + } rc := m.Run() clearRepJobData() + clearScanJobData() if rc != 0 { os.Exit(rc) } @@ -99,6 +111,25 @@ func TestRepJob(t *testing.T) { assert.NotNil(err) } +func TestScanJob(t *testing.T) { + sj := NewScanJob(scanJobID) + assert := assert.New(t) + err := sj.Init() + assert.Nil(err) + assert.Equal(scanJobID, sj.ID()) + assert.Equal(ScanType, sj.Type()) + p := fmt.Sprintf("/var/log/jobs/scan_job/job_%d.log", scanJobID) + assert.Equal(p, sj.LogPath()) + err = sj.UpdateStatus(models.JobRetrying) + assert.Nil(err) + j, err := dao.GetScanJob(scanJobID) + assert.Equal(models.JobRetrying, j.Status) + assert.Equal("sha256:0204dc6e09fa57ab99ac40e415eb637d62c8b2571ecbbc9ca0eb5e2ad2b5c56f", sj.parm.digest) + sj2 := NewScanJob(99999) + err = sj2.Init() + assert.NotNil(err) +} + func TestStatusUpdater(t *testing.T) { assert := assert.New(t) rj := NewRepJob(repJobID) @@ -166,3 +197,25 @@ func clearRepJobData() error { } return nil } + +func prepareScanJobData() error { + if err := clearScanJobData(); err != nil { + return err + } + sj := models.ScanJob{ + Status: models.JobPending, + Repository: "library/ubuntu", + Tag: "15.10", + Digest: "sha256:0204dc6e09fa57ab99ac40e415eb637d62c8b2571ecbbc9ca0eb5e2ad2b5c56f", + } + id, err := dao.AddScanJob(sj) + if err != nil { + return err + } + scanJobID = id + return nil +} + +func clearScanJobData() error { + return dao.ClearTable(models.ScanJobTable) +} diff --git a/src/jobservice/job/jobs.go b/src/jobservice/job/jobs.go index ef33e4ad4..4ecd0dd76 100644 --- a/src/jobservice/job/jobs.go +++ b/src/jobservice/job/jobs.go @@ -20,6 +20,7 @@ import ( "github.com/vmware/harbor/src/jobservice/config" "fmt" + "path/filepath" ) // Type is for job Type @@ -166,3 +167,63 @@ func (rj *RepJob) Init() error { func NewRepJob(id int64) *RepJob { return &RepJob{id: id} } + +//ScanJob implements the Job interface, representing a job for scanning image. +type ScanJob struct { + id int64 + parm *ScanJobParm +} + +//ScanJobParm wraps the parms of a image scan job. +type ScanJobParm struct { + repository string + tag string + digest string +} + +//ID returns the id of the scan +func (sj *ScanJob) ID() int64 { + return sj.id +} + +//Type always return ScanType +func (sj *ScanJob) Type() Type { + return ScanType +} + +//LogPath returns the absolute path of the log file for the job, log files for scan job will be put in a sub folder of base log path. +func (sj *ScanJob) LogPath() string { + return GetJobLogPath(filepath.Join(config.LogDir(), "scan_job"), sj.id) +} + +//String ... +func (sj *ScanJob) String() string { + return fmt.Sprintf("{JobID: %d, JobType: %v}", sj.ID(), sj.Type()) +} + +//UpdateStatus ... +func (sj *ScanJob) UpdateStatus(status string) error { + return dao.UpdateScanJobStatus(sj.id, status) +} + +//Init query the DB and populate the information of the image to scan in the parm of this job. +func (sj *ScanJob) Init() error { + job, err := dao.GetScanJob(sj.id) + if err != nil { + return fmt.Errorf("Failed to get job, error: %v", err) + } + if job == nil { + return fmt.Errorf("The job doesn't exist in DB, job id: %d", sj.id) + } + sj.parm = &ScanJobParm{ + repository: job.Repository, + tag: job.Tag, + digest: job.Digest, + } + return nil +} + +//NewScanJob creates a instance of ScanJob by id. +func NewScanJob(id int64) *ScanJob { + return &ScanJob{id: id} +} diff --git a/src/jobservice/replication/transfer.go b/src/jobservice/replication/transfer.go index 5df465408..4d1677637 100644 --- a/src/jobservice/replication/transfer.go +++ b/src/jobservice/replication/transfer.go @@ -33,6 +33,7 @@ import ( "github.com/vmware/harbor/src/common/utils/registry" "github.com/vmware/harbor/src/common/utils/registry/auth" "github.com/vmware/harbor/src/jobservice/config" + "github.com/vmware/harbor/src/jobservice/utils" ) const ( @@ -136,8 +137,8 @@ func (i *Initializer) Enter() (string, error) { func (i *Initializer) enter() (string, error) { c := &http.Cookie{Name: models.UISecretCookie, Value: i.srcSecret} srcCred := auth.NewCookieCredential(c) - srcClient, err := newRepositoryClient(i.srcURL, i.insecure, srcCred, - config.InternalTokenServiceEndpoint(), i.repository, "repository", i.repository, "pull", "push", "*") + srcClient, err := utils.NewRepositoryClient(i.srcURL, i.insecure, srcCred, + config.InternalTokenServiceEndpoint(), i.repository, "pull", "push", "*") if err != nil { i.logger.Errorf("an error occurred while creating source repository client: %v", err) return "", err @@ -145,8 +146,8 @@ func (i *Initializer) enter() (string, error) { i.srcClient = srcClient dstCred := auth.NewBasicAuthCredential(i.dstUsr, i.dstPwd) - dstClient, err := newRepositoryClient(i.dstURL, i.insecure, dstCred, - "", i.repository, "repository", i.repository, "pull", "push", "*") + dstClient, err := utils.NewRepositoryClient(i.dstURL, i.insecure, dstCred, + "", i.repository, "pull", "push", "*") if err != nil { i.logger.Errorf("an error occurred while creating destination repository client: %v", err) return "", err @@ -450,35 +451,3 @@ func (m *ManifestPusher) enter() (string, error) { return StatePullManifest, nil } - -func newRepositoryClient(endpoint string, insecure bool, credential auth.Credential, - tokenServiceEndpoint, repository, scopeType, scopeName string, - scopeActions ...string) (*registry.Repository, error) { - authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, - tokenServiceEndpoint, scopeType, scopeName, scopeActions...) - - store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer) - if err != nil { - return nil, err - } - - uam := &userAgentModifier{ - userAgent: "harbor-registry-client", - } - - client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store, uam) - if err != nil { - return nil, err - } - return client, nil -} - -type userAgentModifier struct { - userAgent string -} - -// Modify adds user-agent header to the request -func (u *userAgentModifier) Modify(req *http.Request) error { - req.Header.Set(http.CanonicalHeaderKey("User-Agent"), u.userAgent) - return nil -} diff --git a/src/jobservice/router.go b/src/jobservice/router.go index 33c1d86fb..ba88442c4 100644 --- a/src/jobservice/router.go +++ b/src/jobservice/router.go @@ -24,4 +24,5 @@ func initRouters() { beego.Router("/api/jobs/replication", &api.ReplicationJob{}) beego.Router("/api/jobs/replication/:id/log", &api.ReplicationJob{}, "get:GetLog") beego.Router("/api/jobs/replication/actions", &api.ReplicationJob{}, "post:HandleAction") + beego.Router("/api/jobs/scan", &api.ImageScanJob{}) } diff --git a/src/jobservice/utils/utils.go b/src/jobservice/utils/utils.go new file mode 100644 index 000000000..1702d98c4 --- /dev/null +++ b/src/jobservice/utils/utils.go @@ -0,0 +1,53 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "github.com/vmware/harbor/src/common/utils/registry" + "github.com/vmware/harbor/src/common/utils/registry/auth" + "net/http" +) + +//NewRepositoryClient create a repository client with scope type "reopsitory" and scope as the repository it would access. +func NewRepositoryClient(endpoint string, insecure bool, credential auth.Credential, + tokenServiceEndpoint, repository string, actions ...string) (*registry.Repository, error) { + authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, + tokenServiceEndpoint, "repository", repository, actions...) + + store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer) + if err != nil { + return nil, err + } + + uam := &userAgentModifier{ + userAgent: "harbor-registry-client", + } + + client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store, uam) + if err != nil { + return nil, err + } + return client, nil +} + +type userAgentModifier struct { + userAgent string +} + +// Modify adds user-agent header to the request +func (u *userAgentModifier) Modify(req *http.Request) error { + req.Header.Set(http.CanonicalHeaderKey("User-Agent"), u.userAgent) + return nil +}