From 10c857346419419016de42bcb44ae9a05cdfd1f3 Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Mon, 24 Jul 2017 19:50:35 +0800 Subject: [PATCH] provide API for scanning images under a projet --- src/common/dao/repository.go | 11 +++--- src/ui/api/repository.go | 48 +++++++++++++++++-------- src/ui/api/utils.go | 2 +- src/ui/utils/utils.go | 69 +++++++++++++++++++++++------------- 4 files changed, 84 insertions(+), 46 deletions(-) diff --git a/src/common/dao/repository.go b/src/common/dao/repository.go index d5a61cf4c..fc6f75fbf 100644 --- a/src/common/dao/repository.go +++ b/src/common/dao/repository.go @@ -48,9 +48,9 @@ func GetRepositoryByName(name string) (*models.RepoRecord, error) { } // GetAllRepositories ... -func GetAllRepositories() ([]models.RepoRecord, error) { +func GetAllRepositories() ([]*models.RepoRecord, error) { o := GetOrmer() - var repos []models.RepoRecord + var repos []*models.RepoRecord _, err := o.QueryTable("repository"). OrderBy("Name").All(&repos) return repos, err @@ -160,9 +160,10 @@ func GetRepositoriesByProject(projectID int64, name string, if len(name) != 0 { qs = qs.Filter("Name__contains", name) } - - _, err := qs.Limit(limit). - Offset(offset).All(&repositories) + if limit > 0 { + qs = qs.Limit(limit).Offset(offset) + } + _, err := qs.All(&repositories) return repositories, err } diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index 038824d6f..a730d302c 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -19,6 +19,7 @@ import ( "fmt" "io/ioutil" "net/http" + "strconv" "time" "github.com/docker/distribution/manifest/schema1" @@ -741,23 +742,40 @@ func (ra *RepositoryAPI) ScanAll() { ra.HandleUnauthorized() return } - if !ra.SecurityCtx.IsSysAdmin() { - ra.HandleForbidden(ra.SecurityCtx.GetUsername()) - return - } + projectIDStr := ra.GetString("project_id") + if len(projectIDStr) > 0 { //scan images under the project only. + pid, err := strconv.ParseInt(projectIDStr, 10, 64) + if err != nil || pid <= 0 { + ra.HandleBadRequest(fmt.Sprintf("Invalid project_id %s", projectIDStr)) + return + } + if !ra.SecurityCtx.HasAllPerm(pid) { + ra.HandleForbidden(ra.SecurityCtx.GetUsername()) + return + } + if err := uiutils.ScanImagesByProjectID(pid); err != nil { + log.Errorf("Failed triggering scan images in project: %d, error: %v", pid, err) + ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err)) + return + } + } else { //scan all images in Harbor + if !ra.SecurityCtx.IsSysAdmin() { + ra.HandleForbidden(ra.SecurityCtx.GetUsername()) + return + } + if !utils.ScanAllMarker().Check() { + log.Warningf("There is a scan all scheduled at: %v, the request will not be processed.", utils.ScanAllMarker().Next()) + ra.RenderError(http.StatusPreconditionFailed, "Unable handle frequent scan all requests") + return + } - if !utils.ScanAllMarker().Check() { - log.Warningf("There is a scan all scheduled at: %v, the request will not be processed.", utils.ScanAllMarker().Next()) - ra.RenderError(http.StatusPreconditionFailed, "Unable handle frequent scan all requests") - return + if err := uiutils.ScanAllImages(); err != nil { + log.Errorf("Failed triggering scan all images, error: %v", err) + ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err)) + return + } + utils.ScanAllMarker().Mark() } - - if err := uiutils.ScanAllImages(); err != nil { - log.Errorf("Failed triggering scan all images, error: %v", err) - ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err)) - return - } - utils.ScanAllMarker().Mark() ra.Ctx.ResponseWriter.WriteHeader(http.StatusAccepted) } diff --git a/src/ui/api/utils.go b/src/ui/api/utils.go index a639057f9..8d78729b4 100644 --- a/src/ui/api/utils.go +++ b/src/ui/api/utils.go @@ -175,7 +175,7 @@ func SyncRegistry(pm projectmanager.ProjectManager) error { return err } - var repoRecordsInDB []models.RepoRecord + var repoRecordsInDB []*models.RepoRecord repoRecordsInDB, err = dao.GetAllRepositories() if err != nil { log.Errorf("error occurred while getting all registories. %v", err) diff --git a/src/ui/utils/utils.go b/src/ui/utils/utils.go index 025c452d5..2d324de23 100644 --- a/src/ui/utils/utils.go +++ b/src/ui/utils/utils.go @@ -30,7 +30,7 @@ import ( "net/http" ) -// ScanAllImages scans all images of Harbor by submiting jobs to jobservice, the whole process will move one if failed to subit any job of a single image. +// ScanAllImages scans all images of Harbor by submiting jobs to jobservice, the whole process will move on if failed to submit any job of a single image. func ScanAllImages() error { regURL, err := config.RegistryURL() if err != nil { @@ -42,33 +42,52 @@ func ScanAllImages() error { log.Errorf("Failed to list all repositories, error: %v", err) return err } - log.Infof("Rescanning all images.") + log.Infof("Scanning all images on Harbor.") - go func() { - var repoClient *registry.Repository - var err error - var tags []string - for _, r := range repos { - repoClient, err = NewRepositoryClientForUI(regURL, true, "harbor-ui", r.Name, "pull") - if err != nil { - log.Errorf("Failed to initialize client for repository: %s, error: %v, skip scanning", r.Name, err) - continue - } - tags, err = repoClient.ListTag() - if err != nil { - log.Errorf("Failed to get tags for repository: %s, error: %v, skip scanning.", r.Name, err) - continue - } - for _, t := range tags { - if err = TriggerImageScan(r.Name, t); err != nil { - log.Errorf("Failed to scan image with repository: %s, tag: %s, error: %v.", r.Name, t, err) - } else { - log.Debugf("Triggered scan for image with repository: %s, tag: %s", r.Name, t) - } + go scanRepos(repos, regURL) + return nil +} + +// ScanImagesByProjectID scans all images under a projet, the whole process will move on if failed to submit any job of a single image. +func ScanImagesByProjectID(id int64) error { + regURL, err := config.RegistryURL() + if err != nil { + log.Errorf("Failed to load registry url") + return err + } + repos, err := dao.GetRepositoriesByProject(id, "", 0, 0) + if err != nil { + log.Errorf("Failed list repositories in project %d, error: %v", id, err) + return err + } + log.Infof("Scanning all images in project: %d ", id) + go scanRepos(repos, regURL) + return nil +} + +func scanRepos(repos []*models.RepoRecord, regURL string) { + var repoClient *registry.Repository + var err error + var tags []string + for _, r := range repos { + repoClient, err = NewRepositoryClientForUI(regURL, true, "harbor-ui", r.Name, "pull") + if err != nil { + log.Errorf("Failed to initialize client for repository: %s, error: %v, skip scanning", r.Name, err) + continue + } + tags, err = repoClient.ListTag() + if err != nil { + log.Errorf("Failed to get tags for repository: %s, error: %v, skip scanning.", r.Name, err) + continue + } + for _, t := range tags { + if err = TriggerImageScan(r.Name, t); err != nil { + log.Errorf("Failed to scan image with repository: %s, tag: %s, error: %v.", r.Name, t, err) + } else { + log.Debugf("Triggered scan for image with repository: %s, tag: %s", r.Name, t) } } - }() - return nil + } } // RequestAsUI is a shortcut to make a request attach UI secret and send the request.