diff --git a/src/common/job/const.go b/src/common/job/const.go index 9dd31d840..d8fe2b59e 100644 --- a/src/common/job/const.go +++ b/src/common/job/const.go @@ -3,6 +3,8 @@ package job const ( // ImageScanJob is name of scan job it will be used as key to register to job service. ImageScanJob = "IMAGE_SCAN" + // ImageScanAllJob is the name of "scanall" job in job service + ImageScanAllJob = "IMAGE_SCAN_ALL" // ImageTransfer : the name of image transfer job in job service ImageTransfer = "IMAGE_TRANSFER" // ImageDelete : the name of image delete job in job service diff --git a/src/common/security/secret/context.go b/src/common/security/secret/context.go index 63e0f152c..ac4a5b2e5 100644 --- a/src/common/security/secret/context.go +++ b/src/common/security/secret/context.go @@ -71,7 +71,7 @@ func (s *SecurityContext) IsSolutionUser() bool { } // HasReadPerm returns true if the corresponding user of the secret -// is jobservice, otherwise returns false +// is jobservice or core service, otherwise returns false func (s *SecurityContext) HasReadPerm(projectIDOrName interface{}) bool { if s.store == nil { return false @@ -79,14 +79,22 @@ func (s *SecurityContext) HasReadPerm(projectIDOrName interface{}) bool { return s.store.GetUsername(s.secret) == secret.JobserviceUser || s.store.GetUsername(s.secret) == secret.CoreUser } -// HasWritePerm always returns false +// HasWritePerm returns true if the corresponding user of the secret +// is jobservice or core service, otherwise returns false func (s *SecurityContext) HasWritePerm(projectIDOrName interface{}) bool { - return false + if s.store == nil { + return false + } + return s.store.GetUsername(s.secret) == secret.JobserviceUser || s.store.GetUsername(s.secret) == secret.CoreUser } -// HasAllPerm always returns false +// HasAllPerm returns true if the corresponding user of the secret +// is jobservice or core service, otherwise returns false func (s *SecurityContext) HasAllPerm(projectIDOrName interface{}) bool { - return false + if s.store == nil { + return false + } + return s.store.GetUsername(s.secret) == secret.JobserviceUser || s.store.GetUsername(s.secret) == secret.CoreUser } // GetMyProjects ... diff --git a/src/jobservice/job/impl/scan/all.go b/src/jobservice/job/impl/scan/all.go new file mode 100644 index 000000000..cb7598090 --- /dev/null +++ b/src/jobservice/job/impl/scan/all.go @@ -0,0 +1,141 @@ +// Copyright 2018 The Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import ( + "bytes" + "fmt" + "io/ioutil" + + "net/http" + "os" + "strings" + + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/jobservice/env" + "github.com/goharbor/harbor/src/jobservice/job/impl/utils" +) + +// All query the DB and Registry for all image and tags, +// then call Harbor's API to scan each of them. +type All struct { + registryURL string + secret string + tokenServiceEndpoint string + harborAPIEndpoint string + coreClient *http.Client +} + +// MaxFails implements the interface in job/Interface +func (sa *All) MaxFails() uint { + return 1 +} + +// ShouldRetry implements the interface in job/Interface +func (sa *All) ShouldRetry() bool { + return false +} + +// Validate implements the interface in job/Interface +func (sa *All) Validate(params map[string]interface{}) error { + if len(params) > 0 { + return fmt.Errorf("the parms should be empty for scan all job") + } + return nil +} + +// Run implements the interface in job/Interface +func (sa *All) Run(ctx env.JobContext, params map[string]interface{}) error { + logger := ctx.GetLogger() + logger.Info("Scanning all the images in the registry") + err := sa.init(ctx) + if err != nil { + logger.Errorf("Failed to initialize the job handler, error: %v", err) + return err + } + + repos, err := dao.GetRepositories() + if err != nil { + logger.Errorf("Failed to get the list of repositories, error: %v", err) + return err + } + + for _, r := range repos { + repoClient, err := utils.NewRepositoryClientForJobservice(r.Name, sa.registryURL, sa.secret, sa.tokenServiceEndpoint) + if err != nil { + logger.Errorf("Failed to get repo client for repo: %s, error: %v", r.Name, err) + continue + } + tags, err := repoClient.ListTag() + if err != nil { + logger.Errorf("Failed to get tags for repo: %s, error: %v", r.Name, err) + continue + } + for _, t := range tags { + logger.Infof("Calling harbor-core API to scan image, %s:%s", r.Name, t) + resp, err := sa.coreClient.Post(fmt.Sprintf("%s/repositories/%s/tags/%s/scan", sa.harborAPIEndpoint, r.Name, t), + "application/json", + bytes.NewReader([]byte("{}"))) + if err != nil { + logger.Errorf("Failed to trigger image scan, error: %v", err) + } else { + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + logger.Errorf("Failed to read response, error: %v", err) + } else if resp.StatusCode != http.StatusOK { + logger.Errorf("Unexpected response code: %d, data: %v", resp.StatusCode, data) + } + resp.Body.Close() + } + } + + } + + return nil +} + +func (sa *All) init(ctx env.JobContext) error { + if v, err := getAttrFromCtx(ctx, common.RegistryURL); err == nil { + sa.registryURL = v + } else { + return err + } + if v := os.Getenv("JOBSERVICE_SECRET"); len(v) > 0 { + sa.secret = v + } else { + return fmt.Errorf("failed to read evnironment variable JOBSERVICE_SECRET") + } + sa.coreClient, _ = utils.GetClient() + if v, err := getAttrFromCtx(ctx, common.TokenServiceURL); err == nil { + sa.tokenServiceEndpoint = v + } else { + return err + } + if v, err := getAttrFromCtx(ctx, common.CoreURL); err == nil { + v = strings.TrimSuffix(v, "/") + sa.harborAPIEndpoint = v + "/api" + } else { + return err + } + return nil +} + +func getAttrFromCtx(ctx env.JobContext, key string) (string, error) { + if v, ok := ctx.Get(key); ok && len(v.(string)) > 0 { + return v.(string), nil + } + return "", fmt.Errorf("Failed to get required property: %s", key) +} diff --git a/src/jobservice/job/impl/scan/job.go b/src/jobservice/job/impl/scan/clair_job.go similarity index 100% rename from src/jobservice/job/impl/scan/job.go rename to src/jobservice/job/impl/scan/clair_job.go diff --git a/src/jobservice/job/impl/utils/utils.go b/src/jobservice/job/impl/utils/utils.go index dd31bdf5b..9aae48fc2 100644 --- a/src/jobservice/job/impl/utils/utils.go +++ b/src/jobservice/job/impl/utils/utils.go @@ -3,6 +3,8 @@ package utils import ( "fmt" "net/http" + "os" + "sync" "github.com/docker/distribution/registry/auth/token" httpauth "github.com/goharbor/harbor/src/common/http/modifier/auth" @@ -10,6 +12,9 @@ import ( "github.com/goharbor/harbor/src/common/utils/registry/auth" ) +var coreClient *http.Client +var mutex = &sync.Mutex{} + // NewRepositoryClient creates a repository client with standard token authorizer func NewRepositoryClient(endpoint string, insecure bool, credential auth.Credential, tokenServiceEndpoint, repository string) (*registry.Repository, error) { @@ -79,3 +84,20 @@ func GetTokenForRepo(repository, secret, internalTokenServiceURL string) (string return t.Token, nil } + +// GetClient returns the HTTP client that will attach jobservce secret to the request, which can be used for +// accessing Harbor's Core Service. +// This function returns error if the secret of Job service is not set. +func GetClient() (*http.Client, error) { + mutex.Lock() + defer mutex.Unlock() + if coreClient == nil { + secret := os.Getenv("JOBSERVICE_SECRET") + if len(secret) == 0 { + return nil, fmt.Errorf("unable to load secret for job service") + } + modifier := httpauth.NewSecretAuthorizer(secret) + coreClient = &http.Client{Transport: registry.NewTransport(&http.Transport{}, modifier)} + } + return coreClient, nil +} diff --git a/src/jobservice/job/impl/utils/utils_test.go b/src/jobservice/job/impl/utils/utils_test.go new file mode 100644 index 000000000..a950ea228 --- /dev/null +++ b/src/jobservice/job/impl/utils/utils_test.go @@ -0,0 +1,45 @@ +// Copyright 2018 The Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package utils + +import ( + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/goharbor/harbor/src/common/secret" + "github.com/stretchr/testify/assert" +) + +func TestGetClient(t *testing.T) { + assert := assert.New(t) + os.Setenv("", "") + _, err := GetClient() + assert.NotNil(err, "Error should be thrown if secret is not set") + os.Setenv("JOBSERVICE_SECRET", "thesecret") + c, err := GetClient() + assert.Nil(err) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + v := r.Header.Get("Authorization") + assert.Equal(secret.HeaderPrefix+"thesecret", v) + })) + defer ts.Close() + c.Get(ts.URL) + + os.Setenv("", "") + _, err = GetClient() + assert.Nil(err, "Error should be nil once client is initialized") + +} diff --git a/src/jobservice/runtime/bootstrap.go b/src/jobservice/runtime/bootstrap.go index 61dc1d7ae..23ffefd5e 100644 --- a/src/jobservice/runtime/bootstrap.go +++ b/src/jobservice/runtime/bootstrap.go @@ -184,11 +184,12 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool(ctx *env.Context, cfg *config.Con } if err := redisWorkerPool.RegisterJobs( map[string]interface{}{ - job.ImageScanJob: (*scan.ClairJob)(nil), - job.ImageTransfer: (*replication.Transfer)(nil), - job.ImageDelete: (*replication.Deleter)(nil), - job.ImageReplicate: (*replication.Replicator)(nil), - job.ImageGC: (*gc.GarbageCollector)(nil), + job.ImageScanJob: (*scan.ClairJob)(nil), + job.ImageScanAllJob: (*scan.All)(nil), + job.ImageTransfer: (*replication.Transfer)(nil), + job.ImageDelete: (*replication.Deleter)(nil), + job.ImageReplicate: (*replication.Replicator)(nil), + job.ImageGC: (*gc.GarbageCollector)(nil), }); err != nil { // exit return nil, err