diff --git a/src/common/security/secret/context.go b/src/common/security/secret/context.go index 39aa1e4d6..f8e8a30c4 100644 --- a/src/common/security/secret/context.go +++ b/src/common/security/secret/context.go @@ -67,7 +67,7 @@ func (s *SecurityContext) HasReadPerm(projectIDOrName interface{}) bool { if s.store == nil { return false } - return s.store.GetUsername(s.secret) == secret.JobserviceUser + return s.store.GetUsername(s.secret) == secret.JobserviceUser || s.store.GetUsername(s.secret) == secret.UIUser } // HasWritePerm always returns false diff --git a/src/common/utils/registry/repository.go b/src/common/utils/registry/repository.go index d431657e9..b2cebc54c 100644 --- a/src/common/utils/registry/repository.go +++ b/src/common/utils/registry/repository.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "net/http" "net/url" + "sort" "strconv" "strings" // "time" @@ -106,11 +107,19 @@ func (r *Repository) ListTag() ([]string, error) { if err := json.Unmarshal(b, &tagsResp); err != nil { return tags, err } - + sort.Strings(tags) tags = tagsResp.Tags + return tags, nil + } else if resp.StatusCode == http.StatusNotFound { + + // TODO remove the logic if the bug of registry is fixed + // It's a workaround for a bug of registry: when listing tags of + // a repository which is being pushed, a "NAME_UNKNOWN" error will + // been returned, while the catalog API can list this repository. return tags, nil } + return tags, ®istry_error.Error{ StatusCode: resp.StatusCode, Detail: string(b), diff --git a/src/ui/api/replication_job.go b/src/ui/api/replication_job.go index e20c56351..f2f501eb0 100644 --- a/src/ui/api/replication_job.go +++ b/src/ui/api/replication_job.go @@ -25,6 +25,7 @@ import ( "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/utils" ) // RepJobAPI handles request to /api/replicationJobs /api/replicationJobs/:id/log @@ -152,7 +153,7 @@ func (ra *RepJobAPI) GetLog() { log.Errorf("failed to create a request: %v", err) ra.CustomAbort(http.StatusInternalServerError, "") } - addAuthentication(req) + utils.AddUISecret(req) client := &http.Client{} resp, err := client.Do(req) if err != nil { diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index 617d37896..3a23f3c9a 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -19,7 +19,6 @@ import ( "fmt" "io/ioutil" "net/http" - "sort" "time" "github.com/docker/distribution/manifest/schema1" @@ -33,6 +32,7 @@ import ( "github.com/vmware/harbor/src/common/utils/notary" "github.com/vmware/harbor/src/common/utils/registry" "github.com/vmware/harbor/src/ui/config" + uiutils "github.com/vmware/harbor/src/ui/utils" ) // RepositoryAPI handles request to /api/repositories /api/repositories/tags /api/repositories/manifests, the parm has to be put @@ -370,7 +370,7 @@ func (ra *RepositoryAPI) GetTags() { ra.CustomAbort(http.StatusInternalServerError, "internal error") } - tags, err := getSimpleTags(client) + tags, err := client.ListTag() if err != nil { ra.HandleInternalServerError(fmt.Sprintf("failed to get tag of %s: %v", repoName, err)) return @@ -485,31 +485,6 @@ func getV2Manifest(client *registry.Repository, tag string) ( return digest, manifest, config, nil } -// return tag name list for the repository -func getSimpleTags(client *registry.Repository) ([]string, error) { - tags := []string{} - - ts, err := client.ListTag() - if err != nil { - // TODO remove the logic if the bug of registry is fixed - // It's a workaround for a bug of registry: when listing tags of - // a repository which is being pushed, a "NAME_UNKNOWN" error will - // been returned, while the catalog API can list this repository. - - if regErr, ok := err.(*registry_error.Error); ok && - regErr.StatusCode == http.StatusNotFound { - return tags, nil - } - - return nil, err - } - - tags = append(tags, ts...) - sort.Strings(tags) - - return tags, nil -} - // GetManifests returns the manifest of a tag func (ra *RepositoryAPI) GetManifests() { repoName := ra.GetString(":splat") @@ -615,8 +590,8 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo return nil, err } - return NewRepositoryClient(endpoint, true, ra.SecurityCtx.GetUsername(), - repoName, "repository", repoName, "pull", "push", "*") + return uiutils.NewRepositoryClientForUI(endpoint, true, ra.SecurityCtx.GetUsername(), + repoName, "pull", "push", "*") } //GetTopRepos returns the most populor repositories @@ -726,7 +701,7 @@ func (ra *RepositoryAPI) ScanImage() { ra.HandleForbidden(ra.SecurityCtx.GetUsername()) return } - err = TriggerImageScan(repoName, tag) + err = uiutils.TriggerImageScan(repoName, tag) //TODO better check existence if err != nil { log.Errorf("Error while calling job service to trigger image scan: %v", err) @@ -768,7 +743,7 @@ func (ra *RepositoryAPI) VulnerabilityDetails() { ra.HandleInternalServerError(fmt.Sprintf("failed to get the scan overview, error: %v", err)) return } - if overview != nil { + if overview != nil && len(overview.DetailsKey) > 0 { clairClient := clair.NewClient(config.ClairEndpoint(), nil) log.Debugf("The key for getting details: %s", overview.DetailsKey) details, err := clairClient.GetResult(overview.DetailsKey) @@ -782,6 +757,29 @@ func (ra *RepositoryAPI) VulnerabilityDetails() { ra.ServeJSON() } +// ScanAll handles the api to scan all images on Harbor. +func (ra *RepositoryAPI) ScanAll() { + if !config.WithClair() { + log.Warningf("Harbor is not deployed with Clair, it's not possible to scan images.") + ra.RenderError(http.StatusServiceUnavailable, "") + return + } + if !ra.SecurityCtx.IsAuthenticated() { + ra.HandleUnauthorized() + return + } + if !ra.SecurityCtx.IsSysAdmin() { + ra.HandleForbidden(ra.SecurityCtx.GetUsername()) + return + } + if err := uiutils.ScanAllImages(); err != nil { + log.Errorf("Failed triggering scan all images, error: %v", err) + ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err)) + return + } + ra.Ctx.ResponseWriter.WriteHeader(http.StatusAccepted) +} + func getSignatures(repository, username string) (map[string]*notary.Target, error) { targets, err := notary.GetInternalTargets(config.InternalNotaryEndpoint(), username, repository) diff --git a/src/ui/api/search.go b/src/ui/api/search.go index 2e6e91884..24b60bd7a 100644 --- a/src/ui/api/search.go +++ b/src/ui/api/search.go @@ -26,6 +26,7 @@ import ( "github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/config" + uiutils "github.com/vmware/harbor/src/ui/utils" ) // SearchAPI handles requesst to /api/search @@ -157,13 +158,13 @@ func getTags(repository string) ([]string, error) { return nil, err } - client, err := NewRepositoryClient(url, true, - "admin", repository, "repository", repository, "pull") + client, err := uiutils.NewRepositoryClientForUI(url, true, + "admin", repository, "pull") if err != nil { return nil, err } - tags, err := getSimpleTags(client) + tags, err := client.ListTag() if err != nil { return nil, err } diff --git a/src/ui/api/utils.go b/src/ui/api/utils.go index 275157658..a9ad1c364 100644 --- a/src/ui/api/utils.go +++ b/src/ui/api/utils.go @@ -18,7 +18,6 @@ import ( "bytes" "encoding/json" "fmt" - "io" "io/ioutil" "net/http" "sort" @@ -34,6 +33,7 @@ import ( "github.com/vmware/harbor/src/common/utils/registry/auth" "github.com/vmware/harbor/src/ui/config" "github.com/vmware/harbor/src/ui/projectmanager" + uiutils "github.com/vmware/harbor/src/ui/utils" ) //sysadmin has all privileges to all projects @@ -96,7 +96,7 @@ func TriggerReplication(policyID int64, repository string, } url := buildReplicationURL() - return requestAsUI("POST", url, bytes.NewBuffer(b), http.StatusOK) + return uiutils.RequestAsUI("POST", url, bytes.NewBuffer(b), http.StatusOK) } // TriggerReplicationByRepository triggers the replication according to the repository @@ -140,7 +140,7 @@ func postReplicationAction(policyID int64, acton string) error { return err } - addAuthentication(req) + uiutils.AddUISecret(req) client := &http.Client{} @@ -163,15 +163,6 @@ func postReplicationAction(policyID int64, acton string) error { return fmt.Errorf("%d %s", resp.StatusCode, string(b)) } -func addAuthentication(req *http.Request) { - if req != nil { - req.AddCookie(&http.Cookie{ - Name: models.UISecretCookie, - Value: config.UISecret(), - }) - } -} - // SyncRegistry syncs the repositories of registry with database. func SyncRegistry(pm projectmanager.ProjectManager) error { @@ -291,8 +282,8 @@ func diffRepos(reposInRegistry []string, reposInDB []string, if err != nil { return needsAdd, needsDel, err } - client, err := NewRepositoryClient(endpoint, true, - "admin", repoInR, "repository", repoInR, "pull") + client, err := uiutils.NewRepositoryClientForUI(endpoint, true, + "admin", repoInR, "pull") if err != nil { return needsAdd, needsDel, err } @@ -316,8 +307,7 @@ func diffRepos(reposInRegistry []string, reposInDB []string, if err != nil { return needsAdd, needsDel, err } - client, err := NewRepositoryClient(endpoint, true, - "admin", repoInR, "repository", repoInR, "pull") + client, err := uiutils.NewRepositoryClientForUI(endpoint, true, "admin", repoInR, "pull") if err != nil { return needsAdd, needsDel, err } @@ -354,8 +344,7 @@ func diffRepos(reposInRegistry []string, reposInDB []string, log.Errorf("failed to get registry URL: %v", err) continue } - client, err := NewRepositoryClient(endpoint, true, - "admin", repoInR, "repository", repoInR, "pull") + client, err := uiutils.NewRepositoryClientForUI(endpoint, true, "admin", repoInR, "pull") if err != nil { log.Errorf("failed to create repository client: %v", err) continue @@ -411,11 +400,6 @@ func initRegistryClient() (r *registry.Registry, err error) { return registryClient, nil } -func buildScanJobURL() string { - url := config.InternalJobServiceURL() - return fmt.Sprintf("%s/api/jobs/scan", url) -} - func buildReplicationURL() string { url := config.InternalJobServiceURL() return fmt.Sprintf("%s/api/jobs/replication", url) @@ -482,64 +466,6 @@ func NewRegistryClient(endpoint string, insecure bool, username, scopeType, scop return client, nil } -// NewRepositoryClient ... -// TODO need a registry client which accept a raw token as param -func NewRepositoryClient(endpoint string, insecure bool, username, repository, scopeType, scopeName string, - scopeActions ...string) (*registry.Repository, error) { - - authorizer := auth.NewRegistryUsernameTokenAuthorizer(username, scopeType, scopeName, scopeActions...) - - store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer) - if err != nil { - return nil, err - } - - client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store) - if err != nil { - return nil, err - } - return client, nil -} - -// TriggerImageScan triggers an image scan job on jobservice. -func TriggerImageScan(repository string, tag string) error { - data := &models.ImageScanReq{ - Repo: repository, - Tag: tag, - } - b, err := json.Marshal(&data) - if err != nil { - return err - } - url := buildScanJobURL() - return requestAsUI("POST", url, bytes.NewBuffer(b), http.StatusOK) -} - -// Do not use this when you want to handle the response -// TODO: add a response handler to replace expectSC *when needed* -func requestAsUI(method, url string, body io.Reader, expectSC int) error { - req, err := http.NewRequest(method, url, body) - if err != nil { - return err - } - addAuthentication(req) - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != expectSC { - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - return fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b)) - } - return nil -} - // transformVulnerabilities transforms the returned value of Clair API to a list of VulnerabilityItem func transformVulnerabilities(layerWithVuln *models.ClairLayerEnvelope) []*models.VulnerabilityItem { res := []*models.VulnerabilityItem{} diff --git a/src/ui/router.go b/src/ui/router.go index 6a45e05ef..120d3d419 100644 --- a/src/ui/router.go +++ b/src/ui/router.go @@ -73,6 +73,7 @@ func initRouters() { beego.Router("/api/users/:id([0-9]+)/password", &api.UserAPI{}, "put:ChangePassword") beego.Router("/api/internal/syncregistry", &api.InternalAPI{}, "post:SyncRegistry") beego.Router("/api/repositories", &api.RepositoryAPI{}, "get:Get") + beego.Router("/api/repositories/scanAll", &api.RepositoryAPI{}, "post:ScanAll") beego.Router("/api/repositories/*", &api.RepositoryAPI{}, "delete:Delete") beego.Router("/api/repositories/*/tags/:tag", &api.RepositoryAPI{}, "delete:Delete;get:GetTag") beego.Router("/api/repositories/*/tags", &api.RepositoryAPI{}, "get:GetTags") diff --git a/src/ui/service/notification.go b/src/ui/service/notification.go index 7432a1bde..cf1a846b5 100644 --- a/src/ui/service/notification.go +++ b/src/ui/service/notification.go @@ -28,6 +28,7 @@ import ( "github.com/vmware/harbor/src/ui/api" "github.com/vmware/harbor/src/ui/config" "github.com/vmware/harbor/src/ui/projectmanager/pms" + uiutils "github.com/vmware/harbor/src/ui/utils" ) // NotificationHandler handles request on /service/notifications/, which listens to registry's events. @@ -102,7 +103,7 @@ func (n *NotificationHandler) Post() { go api.TriggerReplicationByRepository(pro.ProjectID, repository, []string{tag}, models.RepOpTransfer) if autoScanEnabled(project) { - if err := api.TriggerImageScan(repository, tag); err != nil { + if err := uiutils.TriggerImageScan(repository, tag); err != nil { log.Warningf("Failed to scan image, repository: %s, tag: %s, error: %v", repository, tag, err) } } diff --git a/src/ui/service/utils/utils.go b/src/ui/service/utils/utils.go deleted file mode 100644 index 7fdebf4c8..000000000 --- a/src/ui/service/utils/utils.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2017 VMware, Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package utils contains methods to support security, cache, and webhook functions. -package utils - -import ( - "net/http" - - "github.com/vmware/harbor/src/common/utils/log" -) - -// VerifySecret verifies the UI_SECRET cookie in a http request. -// TODO remove -func VerifySecret(r *http.Request, expectedSecret string) bool { - c, err := r.Cookie("secret") - if err != nil { - log.Warningf("Failed to get secret cookie, error: %v", err) - } - return c != nil && c.Value == expectedSecret -} diff --git a/src/ui/utils/utils.go b/src/ui/utils/utils.go new file mode 100644 index 000000000..2b23d04c5 --- /dev/null +++ b/src/ui/utils/utils.go @@ -0,0 +1,141 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package utils contains methods to support security, cache, and webhook functions. +package utils + +import ( + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/common/utils/registry" + "github.com/vmware/harbor/src/common/utils/registry/auth" + "github.com/vmware/harbor/src/ui/config" + + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" +) + +// ScanAllImages scans all images of Harbor by submiting jobs to jobservice, the whole process will move one if failed to subit any job of a single image. +func ScanAllImages() error { + regURL, err := config.RegistryURL() + if err != nil { + log.Errorf("Failed to load registry url") + return err + } + repos, err := dao.GetAllRepositories() + if err != nil { + log.Errorf("Failed to list all repositories, error: %v", err) + return err + } + log.Infof("Rescanning all images.") + + go func() { + var repoClient *registry.Repository + var err error + var tags []string + for _, r := range repos { + repoClient, err = NewRepositoryClientForUI(regURL, true, "harbor-ui", r.Name, "pull") + if err != nil { + log.Errorf("Failed to initialize client for repository: %s, error: %v, skip scanning", r.Name, err) + continue + } + tags, err = repoClient.ListTag() + if err != nil { + log.Errorf("Failed to get tags for repository: %s, error: %v, skip scanning.", r.Name, err) + continue + } + for _, t := range tags { + if err = TriggerImageScan(r.Name, t); err != nil { + log.Errorf("Failed to scan image with repository: %s, tag: %s, error: %v.", r.Name, t, err) + } else { + log.Debugf("Triggered scan for image with repository: %s, tag: %s", r.Name, t) + } + } + } + }() + return nil +} + +// RequestAsUI is a shortcut to make a request attach UI secret and send the request. +// Do not use this when you want to handle the response +// TODO: add a response handler to replace expectSC *when needed* +func RequestAsUI(method, url string, body io.Reader, expectSC int) error { + req, err := http.NewRequest(method, url, body) + if err != nil { + return err + } + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err + } + AddUISecret(req) + defer resp.Body.Close() + + if resp.StatusCode != expectSC { + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + return fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b)) + } + return nil +} + +//AddUISecret add secret cookie to a request +func AddUISecret(req *http.Request) { + if req != nil { + req.AddCookie(&http.Cookie{ + Name: models.UISecretCookie, + Value: config.UISecret(), + }) + } +} + +// TriggerImageScan triggers an image scan job on jobservice. +func TriggerImageScan(repository string, tag string) error { + data := &models.ImageScanReq{ + Repo: repository, + Tag: tag, + } + b, err := json.Marshal(&data) + if err != nil { + return err + } + url := fmt.Sprintf("%s/api/jobs/scan", config.InternalJobServiceURL()) + return RequestAsUI("POST", url, bytes.NewBuffer(b), http.StatusOK) +} + +// NewRepositoryClientForUI ... +// TODO need a registry client which accept a raw token as param +func NewRepositoryClientForUI(endpoint string, insecure bool, username, repository string, + scopeActions ...string) (*registry.Repository, error) { + + authorizer := auth.NewRegistryUsernameTokenAuthorizer(username, "repository", repository, scopeActions...) + store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer) + if err != nil { + return nil, err + } + + client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store) + if err != nil { + return nil, err + } + return client, nil +} diff --git a/src/ui/service/utils/utils_test.go b/src/ui/utils/utils_test.go similarity index 100% rename from src/ui/service/utils/utils_test.go rename to src/ui/utils/utils_test.go