Merge pull request #2856 from reasonerjt/project-scan-all

Provide API for scanning images under a project

src/common/dao/repository.go

@@ -48,9 +48,9 @@ func GetRepositoryByName(name string) (*models.RepoRecord, error) {
 }
 
 // GetAllRepositories ...
-func GetAllRepositories() ([]models.RepoRecord, error) {
+func GetAllRepositories() ([]*models.RepoRecord, error) {
 	o := GetOrmer()
-	var repos []models.RepoRecord
+	var repos []*models.RepoRecord
 	_, err := o.QueryTable("repository").
 		OrderBy("Name").All(&repos)
 	return repos, err
@@ -160,9 +160,10 @@ func GetRepositoriesByProject(projectID int64, name string,
 	if len(name) != 0 {
 		qs = qs.Filter("Name__contains", name)
 	}
-
+	if limit > 0 {
-	_, err := qs.Limit(limit).
+		qs = qs.Limit(limit).Offset(offset)
-		Offset(offset).All(&repositories)
+	}
+	_, err := qs.All(&repositories)
 
 	return repositories, err
 }

src/ui/api/repository.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"strconv"
 	"time"
 
 	"github.com/docker/distribution/manifest/schema1"
@@ -741,23 +742,40 @@ func (ra *RepositoryAPI) ScanAll() {
 		ra.HandleUnauthorized()
 		return
 	}
-	if !ra.SecurityCtx.IsSysAdmin() {
+	projectIDStr := ra.GetString("project_id")
-		ra.HandleForbidden(ra.SecurityCtx.GetUsername())
+	if len(projectIDStr) > 0 { //scan images under the project only.
-		return
+		pid, err := strconv.ParseInt(projectIDStr, 10, 64)
-	}
+		if err != nil || pid <= 0 {
+			ra.HandleBadRequest(fmt.Sprintf("Invalid project_id %s", projectIDStr))
+			return
+		}
+		if !ra.SecurityCtx.HasAllPerm(pid) {
+			ra.HandleForbidden(ra.SecurityCtx.GetUsername())
+			return
+		}
+		if err := uiutils.ScanImagesByProjectID(pid); err != nil {
+			log.Errorf("Failed triggering scan images in project: %d, error: %v", pid, err)
+			ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err))
+			return
+		}
+	} else { //scan all images in Harbor
+		if !ra.SecurityCtx.IsSysAdmin() {
+			ra.HandleForbidden(ra.SecurityCtx.GetUsername())
+			return
+		}
+		if !utils.ScanAllMarker().Check() {
+			log.Warningf("There is a scan all scheduled at: %v, the request will not be processed.", utils.ScanAllMarker().Next())
+			ra.RenderError(http.StatusPreconditionFailed, "Unable handle frequent scan all requests")
+			return
+		}
 
-	if !utils.ScanAllMarker().Check() {
+		if err := uiutils.ScanAllImages(); err != nil {
-		log.Warningf("There is a scan all scheduled at: %v, the request will not be processed.", utils.ScanAllMarker().Next())
+			log.Errorf("Failed triggering scan all images, error: %v", err)
-		ra.RenderError(http.StatusPreconditionFailed, "Unable handle frequent scan all requests")
+			ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err))
-		return
+			return
+		}
+		utils.ScanAllMarker().Mark()
 	}
-
-	if err := uiutils.ScanAllImages(); err != nil {
-		log.Errorf("Failed triggering scan all images, error: %v", err)
-		ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err))
-		return
-	}
-	utils.ScanAllMarker().Mark()
 	ra.Ctx.ResponseWriter.WriteHeader(http.StatusAccepted)
 }
 

src/ui/api/utils.go

@@ -175,7 +175,7 @@ func SyncRegistry(pm projectmanager.ProjectManager) error {
 		return err
 	}
 
-	var repoRecordsInDB []models.RepoRecord
+	var repoRecordsInDB []*models.RepoRecord
 	repoRecordsInDB, err = dao.GetAllRepositories()
 	if err != nil {
 		log.Errorf("error occurred while getting all registories. %v", err)

src/ui/utils/utils.go

@@ -30,7 +30,7 @@ import (
 	"net/http"
 )
 
-// ScanAllImages scans all images of Harbor by submiting jobs to jobservice, the whole process will move one if failed to subit any job of a single image.
+// ScanAllImages scans all images of Harbor by submiting jobs to jobservice, the whole process will move on if failed to submit any job of a single image.
 func ScanAllImages() error {
 	regURL, err := config.RegistryURL()
 	if err != nil {
@@ -42,33 +42,52 @@ func ScanAllImages() error {
 		log.Errorf("Failed to list all repositories, error: %v", err)
 		return err
 	}
-	log.Infof("Rescanning all images.")
+	log.Infof("Scanning all images on Harbor.")
 
-	go func() {
+	go scanRepos(repos, regURL)
-		var repoClient *registry.Repository
+	return nil
-		var err error
+}
-		var tags []string
+
-		for _, r := range repos {
+// ScanImagesByProjectID scans all images under a projet, the whole process will move on if failed to submit any job of a single image.
-			repoClient, err = NewRepositoryClientForUI(regURL, true, "harbor-ui", r.Name, "pull")
+func ScanImagesByProjectID(id int64) error {
-			if err != nil {
+	regURL, err := config.RegistryURL()
-				log.Errorf("Failed to initialize client for repository: %s, error: %v, skip scanning", r.Name, err)
+	if err != nil {
-				continue
+		log.Errorf("Failed to load registry url")
-			}
+		return err
-			tags, err = repoClient.ListTag()
+	}
-			if err != nil {
+	repos, err := dao.GetRepositoriesByProject(id, "", 0, 0)
-				log.Errorf("Failed to get tags for repository: %s, error: %v, skip scanning.", r.Name, err)
+	if err != nil {
-				continue
+		log.Errorf("Failed list repositories in project %d, error: %v", id, err)
-			}
+		return err
-			for _, t := range tags {
+	}
-				if err = TriggerImageScan(r.Name, t); err != nil {
+	log.Infof("Scanning all images in project: %d ", id)
-					log.Errorf("Failed to scan image with repository: %s, tag: %s, error: %v.", r.Name, t, err)
+	go scanRepos(repos, regURL)
-				} else {
+	return nil
-					log.Debugf("Triggered scan for image with repository: %s, tag: %s", r.Name, t)
+}
-				}
+
+func scanRepos(repos []*models.RepoRecord, regURL string) {
+	var repoClient *registry.Repository
+	var err error
+	var tags []string
+	for _, r := range repos {
+		repoClient, err = NewRepositoryClientForUI(regURL, true, "harbor-ui", r.Name, "pull")
+		if err != nil {
+			log.Errorf("Failed to initialize client for repository: %s, error: %v, skip scanning", r.Name, err)
+			continue
+		}
+		tags, err = repoClient.ListTag()
+		if err != nil {
+			log.Errorf("Failed to get tags for repository: %s, error: %v, skip scanning.", r.Name, err)
+			continue
+		}
+		for _, t := range tags {
+			if err = TriggerImageScan(r.Name, t); err != nil {
+				log.Errorf("Failed to scan image with repository: %s, tag: %s, error: %v.", r.Name, t, err)
+			} else {
+				log.Debugf("Triggered scan for image with repository: %s, tag: %s", r.Name, t)
 			}
 		}
-	}()
+	}
-	return nil
 }
 
 // RequestAsUI is a shortcut to make a request attach UI secret and send the request.