provide an API to scan all images, and some refactory

src/common/security/secret/context.go

@@ -67,7 +67,7 @@ func (s *SecurityContext) HasReadPerm(projectIDOrName interface{}) bool {
 	if s.store == nil {
 		return false
 	}
-	return s.store.GetUsername(s.secret) == secret.JobserviceUser
+	return s.store.GetUsername(s.secret) == secret.JobserviceUser || s.store.GetUsername(s.secret) == secret.UIUser
 }
 
 // HasWritePerm always returns false

src/common/utils/registry/repository.go

@@ -22,6 +22,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	//	"time"
@@ -106,11 +107,19 @@ func (r *Repository) ListTag() ([]string, error) {
 		if err := json.Unmarshal(b, &tagsResp); err != nil {
 			return tags, err
 		}
-
+		sort.Strings(tags)
 		tags = tagsResp.Tags
 
+		return tags, nil
+	} else if resp.StatusCode == http.StatusNotFound {
+
+		// TODO remove the logic if the bug of registry is fixed
+		// It's a workaround for a bug of registry: when listing tags of
+		// a repository which is being pushed, a "NAME_UNKNOWN" error will
+		// been returned, while the catalog API can list this repository.
 		return tags, nil
 	}
+
 	return tags, &registry_error.Error{
 		StatusCode: resp.StatusCode,
 		Detail:     string(b),

src/ui/api/replication_job.go

@@ -25,6 +25,7 @@ import (
 	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/utils/log"
+	"github.com/vmware/harbor/src/ui/utils"
 )
 
 // RepJobAPI handles request to /api/replicationJobs /api/replicationJobs/:id/log
@@ -152,7 +153,7 @@ func (ra *RepJobAPI) GetLog() {
 		log.Errorf("failed to create a request: %v", err)
 		ra.CustomAbort(http.StatusInternalServerError, "")
 	}
-	addAuthentication(req)
+	utils.AddUISecret(req)
 	client := &http.Client{}
 	resp, err := client.Do(req)
 	if err != nil {

src/ui/api/repository.go

@@ -19,7 +19,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"sort"
 	"time"
 
 	"github.com/docker/distribution/manifest/schema1"
@@ -33,6 +32,7 @@ import (
 	"github.com/vmware/harbor/src/common/utils/notary"
 	"github.com/vmware/harbor/src/common/utils/registry"
 	"github.com/vmware/harbor/src/ui/config"
+	uiutils "github.com/vmware/harbor/src/ui/utils"
 )
 
 // RepositoryAPI handles request to /api/repositories /api/repositories/tags /api/repositories/manifests, the parm has to be put
@@ -370,7 +370,7 @@ func (ra *RepositoryAPI) GetTags() {
 		ra.CustomAbort(http.StatusInternalServerError, "internal error")
 	}
 
-	tags, err := getSimpleTags(client)
+	tags, err := client.ListTag()
 	if err != nil {
 		ra.HandleInternalServerError(fmt.Sprintf("failed to get tag of %s: %v", repoName, err))
 		return
@@ -485,31 +485,6 @@ func getV2Manifest(client *registry.Repository, tag string) (
 	return digest, manifest, config, nil
 }
 
-// return tag name list for the repository
-func getSimpleTags(client *registry.Repository) ([]string, error) {
-	tags := []string{}
-
-	ts, err := client.ListTag()
-	if err != nil {
-		// TODO remove the logic if the bug of registry is fixed
-		// It's a workaround for a bug of registry: when listing tags of
-		// a repository which is being pushed, a "NAME_UNKNOWN" error will
-		// been returned, while the catalog API can list this repository.
-
-		if regErr, ok := err.(*registry_error.Error); ok &&
-			regErr.StatusCode == http.StatusNotFound {
-			return tags, nil
-		}
-
-		return nil, err
-	}
-
-	tags = append(tags, ts...)
-	sort.Strings(tags)
-
-	return tags, nil
-}
-
 // GetManifests returns the manifest of a tag
 func (ra *RepositoryAPI) GetManifests() {
 	repoName := ra.GetString(":splat")
@@ -615,8 +590,8 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo
 		return nil, err
 	}
 
-	return NewRepositoryClient(endpoint, true, ra.SecurityCtx.GetUsername(),
+	return uiutils.NewRepositoryClientForUI(endpoint, true, ra.SecurityCtx.GetUsername(),
-		repoName, "repository", repoName, "pull", "push", "*")
+		repoName, "pull", "push", "*")
 }
 
 //GetTopRepos returns the most populor repositories
@@ -726,7 +701,7 @@ func (ra *RepositoryAPI) ScanImage() {
 		ra.HandleForbidden(ra.SecurityCtx.GetUsername())
 		return
 	}
-	err = TriggerImageScan(repoName, tag)
+	err = uiutils.TriggerImageScan(repoName, tag)
 	//TODO better check existence
 	if err != nil {
 		log.Errorf("Error while calling job service to trigger image scan: %v", err)
@@ -768,7 +743,7 @@ func (ra *RepositoryAPI) VulnerabilityDetails() {
 		ra.HandleInternalServerError(fmt.Sprintf("failed to get the scan overview, error: %v", err))
 		return
 	}
-	if overview != nil {
+	if overview != nil && len(overview.DetailsKey) > 0 {
 		clairClient := clair.NewClient(config.ClairEndpoint(), nil)
 		log.Debugf("The key for getting details: %s", overview.DetailsKey)
 		details, err := clairClient.GetResult(overview.DetailsKey)
@@ -782,6 +757,29 @@ func (ra *RepositoryAPI) VulnerabilityDetails() {
 	ra.ServeJSON()
 }
 
+// ScanAll handles the api to scan all images on Harbor.
+func (ra *RepositoryAPI) ScanAll() {
+	if !config.WithClair() {
+		log.Warningf("Harbor is not deployed with Clair, it's not possible to scan images.")
+		ra.RenderError(http.StatusServiceUnavailable, "")
+		return
+	}
+	if !ra.SecurityCtx.IsAuthenticated() {
+		ra.HandleUnauthorized()
+		return
+	}
+	if !ra.SecurityCtx.IsSysAdmin() {
+		ra.HandleForbidden(ra.SecurityCtx.GetUsername())
+		return
+	}
+	if err := uiutils.ScanAllImages(); err != nil {
+		log.Errorf("Failed triggering scan all images, error: %v", err)
+		ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err))
+		return
+	}
+	ra.Ctx.ResponseWriter.WriteHeader(http.StatusAccepted)
+}
+
 func getSignatures(repository, username string) (map[string]*notary.Target, error) {
 	targets, err := notary.GetInternalTargets(config.InternalNotaryEndpoint(),
 		username, repository)

src/ui/api/search.go

@@ -26,6 +26,7 @@ import (
 	"github.com/vmware/harbor/src/common/utils"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/ui/config"
+	uiutils "github.com/vmware/harbor/src/ui/utils"
 )
 
 // SearchAPI handles requesst to /api/search
@@ -157,13 +158,13 @@ func getTags(repository string) ([]string, error) {
 		return nil, err
 	}
 
-	client, err := NewRepositoryClient(url, true,
+	client, err := uiutils.NewRepositoryClientForUI(url, true,
-		"admin", repository, "repository", repository, "pull")
+		"admin", repository, "pull")
 	if err != nil {
 		return nil, err
 	}
 
-	tags, err := getSimpleTags(client)
+	tags, err := client.ListTag()
 	if err != nil {
 		return nil, err
 	}

src/ui/api/utils.go

@@ -18,7 +18,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net/http"
 	"sort"
@@ -34,6 +33,7 @@ import (
 	"github.com/vmware/harbor/src/common/utils/registry/auth"
 	"github.com/vmware/harbor/src/ui/config"
 	"github.com/vmware/harbor/src/ui/projectmanager"
+	uiutils "github.com/vmware/harbor/src/ui/utils"
 )
 
 //sysadmin has all privileges to all projects
@@ -96,7 +96,7 @@ func TriggerReplication(policyID int64, repository string,
 	}
 	url := buildReplicationURL()
 
-	return requestAsUI("POST", url, bytes.NewBuffer(b), http.StatusOK)
+	return uiutils.RequestAsUI("POST", url, bytes.NewBuffer(b), http.StatusOK)
 }
 
 // TriggerReplicationByRepository triggers the replication according to the repository
@@ -140,7 +140,7 @@ func postReplicationAction(policyID int64, acton string) error {
 		return err
 	}
 
-	addAuthentication(req)
+	uiutils.AddUISecret(req)
 
 	client := &http.Client{}
 
@@ -163,15 +163,6 @@ func postReplicationAction(policyID int64, acton string) error {
 	return fmt.Errorf("%d %s", resp.StatusCode, string(b))
 }
 
-func addAuthentication(req *http.Request) {
-	if req != nil {
-		req.AddCookie(&http.Cookie{
-			Name:  models.UISecretCookie,
-			Value: config.UISecret(),
-		})
-	}
-}
-
 // SyncRegistry syncs the repositories of registry with database.
 func SyncRegistry(pm projectmanager.ProjectManager) error {
 
@@ -291,8 +282,8 @@ func diffRepos(reposInRegistry []string, reposInDB []string,
 			if err != nil {
 				return needsAdd, needsDel, err
 			}
-			client, err := NewRepositoryClient(endpoint, true,
+			client, err := uiutils.NewRepositoryClientForUI(endpoint, true,
-				"admin", repoInR, "repository", repoInR, "pull")
+				"admin", repoInR, "pull")
 			if err != nil {
 				return needsAdd, needsDel, err
 			}
@@ -316,8 +307,7 @@ func diffRepos(reposInRegistry []string, reposInDB []string,
 			if err != nil {
 				return needsAdd, needsDel, err
 			}
-			client, err := NewRepositoryClient(endpoint, true,
+			client, err := uiutils.NewRepositoryClientForUI(endpoint, true, "admin", repoInR, "pull")
-				"admin", repoInR, "repository", repoInR, "pull")
 			if err != nil {
 				return needsAdd, needsDel, err
 			}
@@ -354,8 +344,7 @@ func diffRepos(reposInRegistry []string, reposInDB []string,
 			log.Errorf("failed to get registry URL: %v", err)
 			continue
 		}
-		client, err := NewRepositoryClient(endpoint, true,
+		client, err := uiutils.NewRepositoryClientForUI(endpoint, true, "admin", repoInR, "pull")
-			"admin", repoInR, "repository", repoInR, "pull")
 		if err != nil {
 			log.Errorf("failed to create repository client: %v", err)
 			continue
@@ -411,11 +400,6 @@ func initRegistryClient() (r *registry.Registry, err error) {
 	return registryClient, nil
 }
 
-func buildScanJobURL() string {
-	url := config.InternalJobServiceURL()
-	return fmt.Sprintf("%s/api/jobs/scan", url)
-}
-
 func buildReplicationURL() string {
 	url := config.InternalJobServiceURL()
 	return fmt.Sprintf("%s/api/jobs/replication", url)
@@ -482,64 +466,6 @@ func NewRegistryClient(endpoint string, insecure bool, username, scopeType, scop
 	return client, nil
 }
 
-// NewRepositoryClient ...
-// TODO need a registry client which accept a raw token as param
-func NewRepositoryClient(endpoint string, insecure bool, username, repository, scopeType, scopeName string,
-	scopeActions ...string) (*registry.Repository, error) {
-
-	authorizer := auth.NewRegistryUsernameTokenAuthorizer(username, scopeType, scopeName, scopeActions...)
-
-	store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
-	if err != nil {
-		return nil, err
-	}
-
-	client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store)
-	if err != nil {
-		return nil, err
-	}
-	return client, nil
-}
-
-// TriggerImageScan triggers an image scan job on jobservice.
-func TriggerImageScan(repository string, tag string) error {
-	data := &models.ImageScanReq{
-		Repo: repository,
-		Tag:  tag,
-	}
-	b, err := json.Marshal(&data)
-	if err != nil {
-		return err
-	}
-	url := buildScanJobURL()
-	return requestAsUI("POST", url, bytes.NewBuffer(b), http.StatusOK)
-}
-
-// Do not use this when you want to handle the response
-// TODO: add a response handler to replace expectSC *when needed*
-func requestAsUI(method, url string, body io.Reader, expectSC int) error {
-	req, err := http.NewRequest(method, url, body)
-	if err != nil {
-		return err
-	}
-	addAuthentication(req)
-	client := &http.Client{}
-	resp, err := client.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != expectSC {
-		b, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return err
-		}
-		return fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
-	}
-	return nil
-}
-
 // transformVulnerabilities transforms the returned value of Clair API to a list of VulnerabilityItem
 func transformVulnerabilities(layerWithVuln *models.ClairLayerEnvelope) []*models.VulnerabilityItem {
 	res := []*models.VulnerabilityItem{}

src/ui/router.go

@@ -73,6 +73,7 @@ func initRouters() {
 	beego.Router("/api/users/:id([0-9]+)/password", &api.UserAPI{}, "put:ChangePassword")
 	beego.Router("/api/internal/syncregistry", &api.InternalAPI{}, "post:SyncRegistry")
 	beego.Router("/api/repositories", &api.RepositoryAPI{}, "get:Get")
+	beego.Router("/api/repositories/scanAll", &api.RepositoryAPI{}, "post:ScanAll")
 	beego.Router("/api/repositories/*", &api.RepositoryAPI{}, "delete:Delete")
 	beego.Router("/api/repositories/*/tags/:tag", &api.RepositoryAPI{}, "delete:Delete;get:GetTag")
 	beego.Router("/api/repositories/*/tags", &api.RepositoryAPI{}, "get:GetTags")

src/ui/service/notification.go

@@ -28,6 +28,7 @@ import (
 	"github.com/vmware/harbor/src/ui/api"
 	"github.com/vmware/harbor/src/ui/config"
 	"github.com/vmware/harbor/src/ui/projectmanager/pms"
+	uiutils "github.com/vmware/harbor/src/ui/utils"
 )
 
 // NotificationHandler handles request on /service/notifications/, which listens to registry's events.
@@ -102,7 +103,7 @@ func (n *NotificationHandler) Post() {
 
 			go api.TriggerReplicationByRepository(pro.ProjectID, repository, []string{tag}, models.RepOpTransfer)
 			if autoScanEnabled(project) {
-				if err := api.TriggerImageScan(repository, tag); err != nil {
+				if err := uiutils.TriggerImageScan(repository, tag); err != nil {
 					log.Warningf("Failed to scan image, repository: %s, tag: %s, error: %v", repository, tag, err)
 				}
 			}

src/ui/service/utils/utils.go

@@ -1,32 +0,0 @@
-// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package utils contains methods to support security, cache, and webhook functions.
-package utils
-
-import (
-	"net/http"
-
-	"github.com/vmware/harbor/src/common/utils/log"
-)
-
-// VerifySecret verifies the UI_SECRET cookie in a http request.
-// TODO remove
-func VerifySecret(r *http.Request, expectedSecret string) bool {
-	c, err := r.Cookie("secret")
-	if err != nil {
-		log.Warningf("Failed to get secret cookie, error: %v", err)
-	}
-	return c != nil && c.Value == expectedSecret
-}

src/ui/utils/utils.go

@@ -0,0 +1,141 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package utils contains methods to support security, cache, and webhook functions.
+package utils
+
+import (
+	"github.com/vmware/harbor/src/common/dao"
+	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils/log"
+	"github.com/vmware/harbor/src/common/utils/registry"
+	"github.com/vmware/harbor/src/common/utils/registry/auth"
+	"github.com/vmware/harbor/src/ui/config"
+
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+)
+
+// ScanAllImages scans all images of Harbor by submiting jobs to jobservice, the whole process will move one if failed to subit any job of a single image.
+func ScanAllImages() error {
+	regURL, err := config.RegistryURL()
+	if err != nil {
+		log.Errorf("Failed to load registry url")
+		return err
+	}
+	repos, err := dao.GetAllRepositories()
+	if err != nil {
+		log.Errorf("Failed to list all repositories, error: %v", err)
+		return err
+	}
+	log.Infof("Rescanning all images.")
+
+	go func() {
+		var repoClient *registry.Repository
+		var err error
+		var tags []string
+		for _, r := range repos {
+			repoClient, err = NewRepositoryClientForUI(regURL, true, "harbor-ui", r.Name, "pull")
+			if err != nil {
+				log.Errorf("Failed to initialize client for repository: %s, error: %v, skip scanning", r.Name, err)
+				continue
+			}
+			tags, err = repoClient.ListTag()
+			if err != nil {
+				log.Errorf("Failed to get tags for repository: %s, error: %v, skip scanning.", r.Name, err)
+				continue
+			}
+			for _, t := range tags {
+				if err = TriggerImageScan(r.Name, t); err != nil {
+					log.Errorf("Failed to scan image with repository: %s, tag: %s, error: %v.", r.Name, t, err)
+				} else {
+					log.Debugf("Triggered scan for image with repository: %s, tag: %s", r.Name, t)
+				}
+			}
+		}
+	}()
+	return nil
+}
+
+// RequestAsUI is a shortcut to make a request attach UI secret and send the request.
+// Do not use this when you want to handle the response
+// TODO: add a response handler to replace expectSC *when needed*
+func RequestAsUI(method, url string, body io.Reader, expectSC int) error {
+	req, err := http.NewRequest(method, url, body)
+	if err != nil {
+		return err
+	}
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	AddUISecret(req)
+	defer resp.Body.Close()
+
+	if resp.StatusCode != expectSC {
+		b, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		return fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
+	}
+	return nil
+}
+
+//AddUISecret add secret cookie to a request
+func AddUISecret(req *http.Request) {
+	if req != nil {
+		req.AddCookie(&http.Cookie{
+			Name:  models.UISecretCookie,
+			Value: config.UISecret(),
+		})
+	}
+}
+
+// TriggerImageScan triggers an image scan job on jobservice.
+func TriggerImageScan(repository string, tag string) error {
+	data := &models.ImageScanReq{
+		Repo: repository,
+		Tag:  tag,
+	}
+	b, err := json.Marshal(&data)
+	if err != nil {
+		return err
+	}
+	url := fmt.Sprintf("%s/api/jobs/scan", config.InternalJobServiceURL())
+	return RequestAsUI("POST", url, bytes.NewBuffer(b), http.StatusOK)
+}
+
+// NewRepositoryClientForUI ...
+// TODO need a registry client which accept a raw token as param
+func NewRepositoryClientForUI(endpoint string, insecure bool, username, repository string,
+	scopeActions ...string) (*registry.Repository, error) {
+
+	authorizer := auth.NewRegistryUsernameTokenAuthorizer(username, "repository", repository, scopeActions...)
+	store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}