mirror of
https://github.com/goharbor/harbor.git
synced 2024-10-02 23:37:38 +02:00
Merge pull request #748 from ywk253100/repo_in_db
Enhancements and refactors due to storing repositories in database
This commit is contained in:
commit
ce7d9fcefb
@ -62,7 +62,7 @@ create table project (
|
|||||||
project_id int NOT NULL AUTO_INCREMENT,
|
project_id int NOT NULL AUTO_INCREMENT,
|
||||||
owner_id int NOT NULL,
|
owner_id int NOT NULL,
|
||||||
# The max length of name controlled by API is 30,
|
# The max length of name controlled by API is 30,
|
||||||
# and 11 bytes is reserved for marking the deleted project.
|
# and 11 is reserved for marking the deleted project.
|
||||||
name varchar (41) NOT NULL,
|
name varchar (41) NOT NULL,
|
||||||
creation_time timestamp,
|
creation_time timestamp,
|
||||||
update_time timestamp,
|
update_time timestamp,
|
||||||
|
@ -290,7 +290,14 @@ func (p *ProjectAPI) List() {
|
|||||||
projectList[i].Togglable = true
|
projectList[i].Togglable = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
projectList[i].RepoCount = getRepoCountByProject(projectList[i].Name)
|
|
||||||
|
repos, err := dao.GetRepositoryByProjectName(projectList[i].Name)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("failed to get repositories of project %s: %v", projectList[i].Name, err)
|
||||||
|
p.CustomAbort(http.StatusInternalServerError, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
projectList[i].RepoCount = len(repos)
|
||||||
}
|
}
|
||||||
|
|
||||||
p.setPaginationHeader(total, page, pageSize)
|
p.setPaginationHeader(total, page, pageSize)
|
||||||
|
@ -21,8 +21,6 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/docker/distribution/manifest/schema1"
|
"github.com/docker/distribution/manifest/schema1"
|
||||||
"github.com/docker/distribution/manifest/schema2"
|
"github.com/docker/distribution/manifest/schema2"
|
||||||
@ -35,6 +33,7 @@ import (
|
|||||||
|
|
||||||
registry_error "github.com/vmware/harbor/utils/registry/error"
|
registry_error "github.com/vmware/harbor/utils/registry/error"
|
||||||
|
|
||||||
|
"github.com/vmware/harbor/utils"
|
||||||
"github.com/vmware/harbor/utils/registry/auth"
|
"github.com/vmware/harbor/utils/registry/auth"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -108,7 +107,7 @@ func (ra *RepositoryAPI) Delete() {
|
|||||||
ra.CustomAbort(http.StatusBadRequest, "repo_name is nil")
|
ra.CustomAbort(http.StatusBadRequest, "repo_name is nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
projectName := getProjectName(repoName)
|
projectName, _ := utils.ParseRepository(repoName)
|
||||||
project, err := dao.GetProjectByName(projectName)
|
project, err := dao.GetProjectByName(projectName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("failed to get project %s: %v", projectName, err)
|
log.Errorf("failed to get project %s: %v", projectName, err)
|
||||||
@ -182,6 +181,18 @@ func (ra *RepositoryAPI) Delete() {
|
|||||||
}(t)
|
}(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
exist, err := repositoryExist(repoName, rc)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("failed to check the existence of repository %s: %v", repoName, err)
|
||||||
|
ra.CustomAbort(http.StatusInternalServerError, "")
|
||||||
|
}
|
||||||
|
if !exist {
|
||||||
|
if err = dao.DeleteRepository(repoName); err != nil {
|
||||||
|
log.Errorf("failed to delete repository %s: %v", repoName, err)
|
||||||
|
ra.CustomAbort(http.StatusInternalServerError, "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
log.Debug("refreshing catalog cache")
|
log.Debug("refreshing catalog cache")
|
||||||
if err := cache.RefreshCatalogCache(); err != nil {
|
if err := cache.RefreshCatalogCache(); err != nil {
|
||||||
@ -202,7 +213,7 @@ func (ra *RepositoryAPI) GetTags() {
|
|||||||
ra.CustomAbort(http.StatusBadRequest, "repo_name is nil")
|
ra.CustomAbort(http.StatusBadRequest, "repo_name is nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
projectName := getProjectName(repoName)
|
projectName, _ := utils.ParseRepository(repoName)
|
||||||
project, err := dao.GetProjectByName(projectName)
|
project, err := dao.GetProjectByName(projectName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("failed to get project %s: %v", projectName, err)
|
log.Errorf("failed to get project %s: %v", projectName, err)
|
||||||
@ -270,7 +281,7 @@ func (ra *RepositoryAPI) GetManifests() {
|
|||||||
ra.CustomAbort(http.StatusBadRequest, "version should be v1 or v2")
|
ra.CustomAbort(http.StatusBadRequest, "version should be v1 or v2")
|
||||||
}
|
}
|
||||||
|
|
||||||
projectName := getProjectName(repoName)
|
projectName, _ := utils.ParseRepository(repoName)
|
||||||
project, err := dao.GetProjectByName(projectName)
|
project, err := dao.GetProjectByName(projectName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("failed to get project %s: %v", projectName, err)
|
log.Errorf("failed to get project %s: %v", projectName, err)
|
||||||
@ -397,25 +408,14 @@ func (ra *RepositoryAPI) getUsername() (string, error) {
|
|||||||
|
|
||||||
//GetTopRepos handles request GET /api/repositories/top
|
//GetTopRepos handles request GET /api/repositories/top
|
||||||
func (ra *RepositoryAPI) GetTopRepos() {
|
func (ra *RepositoryAPI) GetTopRepos() {
|
||||||
var err error
|
count, err := ra.GetInt("count", 10)
|
||||||
var countNum int
|
if err != nil || count <= 0 {
|
||||||
count := ra.GetString("count")
|
ra.CustomAbort(http.StatusBadRequest, "invalid count")
|
||||||
if len(count) == 0 {
|
}
|
||||||
countNum = 10
|
|
||||||
} else {
|
repos, err := dao.GetTopRepos(count)
|
||||||
countNum, err = strconv.Atoi(count)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("Get parameters error--count, err: %v", err)
|
log.Errorf("failed to get top repos: %v", err)
|
||||||
ra.CustomAbort(http.StatusBadRequest, "bad request of count")
|
|
||||||
}
|
|
||||||
if countNum <= 0 {
|
|
||||||
log.Warning("count must be a positive integer")
|
|
||||||
ra.CustomAbort(http.StatusBadRequest, "count is 0 or negative")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
repos, err := dao.GetTopRepos(countNum)
|
|
||||||
if err != nil {
|
|
||||||
log.Errorf("error occured in get top 10 repos: %v", err)
|
|
||||||
ra.CustomAbort(http.StatusInternalServerError, "internal server error")
|
ra.CustomAbort(http.StatusInternalServerError, "internal server error")
|
||||||
}
|
}
|
||||||
ra.Data["json"] = repos
|
ra.Data["json"] = repos
|
||||||
@ -439,11 +439,3 @@ func newRepositoryClient(endpoint string, insecure bool, username, password, rep
|
|||||||
}
|
}
|
||||||
return client, nil
|
return client, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getProjectName(repository string) string {
|
|
||||||
project := ""
|
|
||||||
if strings.Contains(repository, "/") {
|
|
||||||
project = repository[0:strings.LastIndex(repository, "/")]
|
|
||||||
}
|
|
||||||
return project
|
|
||||||
}
|
|
||||||
|
124
api/statistic.go
124
api/statistic.go
@ -17,14 +17,26 @@ package api
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/vmware/harbor/dao"
|
"github.com/vmware/harbor/dao"
|
||||||
"github.com/vmware/harbor/models"
|
|
||||||
"github.com/vmware/harbor/service/cache"
|
|
||||||
"github.com/vmware/harbor/utils/log"
|
"github.com/vmware/harbor/utils/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// MPC : count of my projects
|
||||||
|
MPC = "my_project_count"
|
||||||
|
// MRC : count of my repositories
|
||||||
|
MRC = "my_repo_count"
|
||||||
|
// PPC : count of public projects
|
||||||
|
PPC = "public_project_count"
|
||||||
|
// PRC : count of public repositories
|
||||||
|
PRC = "public_repo_count"
|
||||||
|
// TPC : total count of projects
|
||||||
|
TPC = "total_project_count"
|
||||||
|
// TRC : total count of repositories
|
||||||
|
TRC = "total_repo_count"
|
||||||
|
)
|
||||||
|
|
||||||
// StatisticAPI handles request to /api/statistics/
|
// StatisticAPI handles request to /api/statistics/
|
||||||
type StatisticAPI struct {
|
type StatisticAPI struct {
|
||||||
BaseAPI
|
BaseAPI
|
||||||
@ -38,80 +50,60 @@ func (s *StatisticAPI) Prepare() {
|
|||||||
|
|
||||||
// Get total projects and repos of the user
|
// Get total projects and repos of the user
|
||||||
func (s *StatisticAPI) Get() {
|
func (s *StatisticAPI) Get() {
|
||||||
|
statistic := map[string]int64{}
|
||||||
|
|
||||||
|
n, err := dao.GetTotalOfProjects("", 1)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("failed to get total of public projects: %v", err)
|
||||||
|
s.CustomAbort(http.StatusInternalServerError, "")
|
||||||
|
}
|
||||||
|
statistic[PPC] = n
|
||||||
|
|
||||||
|
n, err = dao.GetTotalOfPublicRepositories("")
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("failed to get total of public repositories: %v", err)
|
||||||
|
s.CustomAbort(http.StatusInternalServerError, "")
|
||||||
|
}
|
||||||
|
statistic[PRC] = n
|
||||||
|
|
||||||
isAdmin, err := dao.IsAdminRole(s.userID)
|
isAdmin, err := dao.IsAdminRole(s.userID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("Error occured in check admin, error: %v", err)
|
log.Errorf("Error occured in check admin, error: %v", err)
|
||||||
s.CustomAbort(http.StatusInternalServerError, "Internal error.")
|
s.CustomAbort(http.StatusInternalServerError, "Internal error.")
|
||||||
}
|
}
|
||||||
var projectList []models.Project
|
|
||||||
if isAdmin {
|
if isAdmin {
|
||||||
projectList, err = dao.GetProjects("")
|
n, err := dao.GetTotalOfProjects("")
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("failed to get total of projects: %v", err)
|
||||||
|
s.CustomAbort(http.StatusInternalServerError, "")
|
||||||
|
}
|
||||||
|
statistic[MPC] = n
|
||||||
|
statistic[TPC] = n
|
||||||
|
|
||||||
|
n, err = dao.GetTotalOfRepositories("")
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("failed to get total of repositories: %v", err)
|
||||||
|
s.CustomAbort(http.StatusInternalServerError, "")
|
||||||
|
}
|
||||||
|
statistic[MRC] = n
|
||||||
|
statistic[TRC] = n
|
||||||
} else {
|
} else {
|
||||||
projectList, err = dao.GetUserRelevantProjects(s.userID, "")
|
n, err := dao.GetTotalOfUserRelevantProjects(s.userID, "")
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("Error occured in QueryProject, error: %v", err)
|
log.Errorf("failed to get total of projects for user %d: %v", s.userID, err)
|
||||||
s.CustomAbort(http.StatusInternalServerError, "Internal error.")
|
s.CustomAbort(http.StatusInternalServerError, "")
|
||||||
}
|
}
|
||||||
proMap := map[string]int{}
|
statistic[MPC] = n
|
||||||
proMap["my_project_count"] = 0
|
|
||||||
proMap["my_repo_count"] = 0
|
n, err = dao.GetTotalOfUserRelevantRepositories(s.userID, "")
|
||||||
proMap["public_project_count"] = 0
|
|
||||||
proMap["public_repo_count"] = 0
|
|
||||||
var publicProjects []models.Project
|
|
||||||
publicProjects, err = dao.GetProjects("", 1)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("Error occured in QueryPublicProject, error: %v", err)
|
log.Errorf("failed to get total of repositories for user %d: %v", s.userID, err)
|
||||||
s.CustomAbort(http.StatusInternalServerError, "Internal error.")
|
s.CustomAbort(http.StatusInternalServerError, "")
|
||||||
}
|
}
|
||||||
proMap["public_project_count"] = len(publicProjects)
|
statistic[MRC] = n
|
||||||
for i := 0; i < len(publicProjects); i++ {
|
|
||||||
proMap["public_repo_count"] += getRepoCountByProject(publicProjects[i].Name)
|
|
||||||
}
|
}
|
||||||
if isAdmin {
|
|
||||||
proMap["total_project_count"] = len(projectList)
|
s.Data["json"] = statistic
|
||||||
proMap["total_repo_count"] = getTotalRepoCount()
|
|
||||||
}
|
|
||||||
for i := 0; i < len(projectList); i++ {
|
|
||||||
if isAdmin {
|
|
||||||
projectList[i].Role = models.PROJECTADMIN
|
|
||||||
}
|
|
||||||
if projectList[i].Role == models.PROJECTADMIN || projectList[i].Role == models.DEVELOPER ||
|
|
||||||
projectList[i].Role == models.GUEST {
|
|
||||||
proMap["my_project_count"]++
|
|
||||||
proMap["my_repo_count"] += getRepoCountByProject(projectList[i].Name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s.Data["json"] = proMap
|
|
||||||
s.ServeJSON()
|
s.ServeJSON()
|
||||||
}
|
}
|
||||||
|
|
||||||
//getReposByProject returns repo numbers of specified project
|
|
||||||
func getRepoCountByProject(projectName string) int {
|
|
||||||
repoList, err := cache.GetRepoFromCache()
|
|
||||||
if err != nil {
|
|
||||||
log.Errorf("Failed to get repo from cache, error: %v", err)
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
var resp int
|
|
||||||
if len(projectName) > 0 {
|
|
||||||
for _, r := range repoList {
|
|
||||||
if strings.Contains(r, "/") && r[0:strings.LastIndex(r, "/")] == projectName {
|
|
||||||
resp++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return resp
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
//getTotalRepoCount returns total repo count
|
|
||||||
func getTotalRepoCount() int {
|
|
||||||
repoList, err := cache.GetRepoFromCache()
|
|
||||||
if err != nil {
|
|
||||||
log.Errorf("Failed to get repo from cache, error: %v", err)
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return len(repoList)
|
|
||||||
|
|
||||||
}
|
|
||||||
|
@ -2,13 +2,17 @@ package api
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/vmware/harbor/tests/apitests/apilib"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/vmware/harbor/tests/apitests/apilib"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestStatisticGet(t *testing.T) {
|
func TestStatisticGet(t *testing.T) {
|
||||||
|
if err := SyncRegistry(); err != nil {
|
||||||
|
t.Fatalf("failed to sync repositories from registry: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
fmt.Println("Testing Statistic API")
|
fmt.Println("Testing Statistic API")
|
||||||
assert := assert.New(t)
|
assert := assert.New(t)
|
||||||
|
116
api/utils.go
116
api/utils.go
@ -33,6 +33,7 @@ import (
|
|||||||
"github.com/vmware/harbor/utils"
|
"github.com/vmware/harbor/utils"
|
||||||
"github.com/vmware/harbor/utils/log"
|
"github.com/vmware/harbor/utils/log"
|
||||||
"github.com/vmware/harbor/utils/registry"
|
"github.com/vmware/harbor/utils/registry"
|
||||||
|
registry_error "github.com/vmware/harbor/utils/registry/error"
|
||||||
)
|
)
|
||||||
|
|
||||||
func checkProjectPermission(userID int, projectID int64) bool {
|
func checkProjectPermission(userID int, projectID int64) bool {
|
||||||
@ -243,13 +244,8 @@ func addAuthentication(req *http.Request) {
|
|||||||
func SyncRegistry() error {
|
func SyncRegistry() error {
|
||||||
|
|
||||||
log.Debugf("Start syncing repositories from registry to DB... ")
|
log.Debugf("Start syncing repositories from registry to DB... ")
|
||||||
rc, err := initRegistryClient()
|
|
||||||
if err != nil {
|
|
||||||
log.Errorf("error occurred while initializing registry client for %v", err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
reposInRegistry, err := rc.Catalog()
|
reposInRegistry, err := catalog()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(err)
|
log.Error(err)
|
||||||
return err
|
return err
|
||||||
@ -261,6 +257,7 @@ func SyncRegistry() error {
|
|||||||
log.Errorf("error occurred while getting all registories. %v", err)
|
log.Errorf("error occurred while getting all registories. %v", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
var reposInDB []string
|
var reposInDB []string
|
||||||
for _, repoRecordInDB := range repoRecordsInDB {
|
for _, repoRecordInDB := range repoRecordsInDB {
|
||||||
reposInDB = append(reposInDB, repoRecordInDB.Name)
|
reposInDB = append(reposInDB, repoRecordInDB.Name)
|
||||||
@ -269,7 +266,6 @@ func SyncRegistry() error {
|
|||||||
var reposToAdd []string
|
var reposToAdd []string
|
||||||
var reposToDel []string
|
var reposToDel []string
|
||||||
reposToAdd, reposToDel = diffRepos(reposInRegistry, reposInDB)
|
reposToAdd, reposToDel = diffRepos(reposInRegistry, reposInDB)
|
||||||
|
|
||||||
if len(reposToAdd) > 0 {
|
if len(reposToAdd) > 0 {
|
||||||
log.Debugf("Start adding repositories into DB... ")
|
log.Debugf("Start adding repositories into DB... ")
|
||||||
for _, repoToAdd := range reposToAdd {
|
for _, repoToAdd := range reposToAdd {
|
||||||
@ -307,6 +303,44 @@ func SyncRegistry() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func catalog() ([]string, error) {
|
||||||
|
repositories := []string{}
|
||||||
|
|
||||||
|
rc, err := initRegistryClient()
|
||||||
|
if err != nil {
|
||||||
|
return repositories, err
|
||||||
|
}
|
||||||
|
|
||||||
|
repos, err := rc.Catalog()
|
||||||
|
if err != nil {
|
||||||
|
return repositories, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, repo := range repos {
|
||||||
|
// TODO remove the workaround when the bug of registry is fixed
|
||||||
|
// TODO read it from config
|
||||||
|
endpoint := os.Getenv("REGISTRY_URL")
|
||||||
|
client, err := cache.NewRepositoryClient(endpoint, true,
|
||||||
|
"admin", repo, "repository", repo)
|
||||||
|
if err != nil {
|
||||||
|
return repositories, err
|
||||||
|
}
|
||||||
|
|
||||||
|
exist, err := repositoryExist(repo, client)
|
||||||
|
if err != nil {
|
||||||
|
return repositories, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !exist {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
repositories = append(repositories, repo)
|
||||||
|
}
|
||||||
|
|
||||||
|
return repositories, nil
|
||||||
|
}
|
||||||
|
|
||||||
func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string) {
|
func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string) {
|
||||||
var needsAdd []string
|
var needsAdd []string
|
||||||
var needsDel []string
|
var needsDel []string
|
||||||
@ -315,13 +349,26 @@ func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string
|
|||||||
sort.Strings(reposInDB)
|
sort.Strings(reposInDB)
|
||||||
|
|
||||||
i, j := 0, 0
|
i, j := 0, 0
|
||||||
|
repoInR, repoInD := "", ""
|
||||||
for i < len(reposInRegistry) && j < len(reposInDB) {
|
for i < len(reposInRegistry) && j < len(reposInDB) {
|
||||||
d := strings.Compare(reposInRegistry[i], reposInDB[j])
|
repoInR = reposInRegistry[i]
|
||||||
|
repoInD = reposInDB[j]
|
||||||
|
d := strings.Compare(repoInR, repoInD)
|
||||||
if d < 0 {
|
if d < 0 {
|
||||||
needsAdd = append(needsAdd, reposInRegistry[i])
|
|
||||||
i++
|
i++
|
||||||
|
exist, err := projectExists(repoInR)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("failed to check the existence of project %s: %v", repoInR, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if !exist {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
needsAdd = append(needsAdd, repoInR)
|
||||||
} else if d > 0 {
|
} else if d > 0 {
|
||||||
needsDel = append(needsDel, reposInDB[j])
|
needsDel = append(needsDel, repoInD)
|
||||||
j++
|
j++
|
||||||
} else {
|
} else {
|
||||||
i++
|
i++
|
||||||
@ -330,8 +377,18 @@ func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string
|
|||||||
}
|
}
|
||||||
|
|
||||||
for i < len(reposInRegistry) {
|
for i < len(reposInRegistry) {
|
||||||
needsAdd = append(needsAdd, reposInRegistry[i])
|
repoInR = reposInRegistry[i]
|
||||||
i++
|
i++
|
||||||
|
exist, err := projectExists(repoInR)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("failed to check whether project of %s exists: %v", repoInR, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if !exist {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
needsAdd = append(needsAdd, repoInR)
|
||||||
}
|
}
|
||||||
|
|
||||||
for j < len(reposInDB) {
|
for j < len(reposInDB) {
|
||||||
@ -342,10 +399,15 @@ func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string
|
|||||||
return needsAdd, needsDel
|
return needsAdd, needsDel
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func projectExists(repository string) (bool, error) {
|
||||||
|
project, _ := utils.ParseRepository(repository)
|
||||||
|
return dao.ProjectExists(project)
|
||||||
|
}
|
||||||
|
|
||||||
func initRegistryClient() (r *registry.Registry, err error) {
|
func initRegistryClient() (r *registry.Registry, err error) {
|
||||||
endpoint := os.Getenv("REGISTRY_URL")
|
endpoint := os.Getenv("REGISTRY_URL")
|
||||||
|
|
||||||
addr := ""
|
addr := endpoint
|
||||||
if strings.Contains(endpoint, "/") {
|
if strings.Contains(endpoint, "/") {
|
||||||
addr = endpoint[strings.LastIndex(endpoint, "/")+1:]
|
addr = endpoint[strings.LastIndex(endpoint, "/")+1:]
|
||||||
}
|
}
|
||||||
@ -409,25 +471,20 @@ func getJobServiceURL() string {
|
|||||||
func getReposByProject(name string, keyword ...string) ([]string, error) {
|
func getReposByProject(name string, keyword ...string) ([]string, error) {
|
||||||
repositories := []string{}
|
repositories := []string{}
|
||||||
|
|
||||||
list, err := getAllRepos()
|
repos, err := dao.GetRepositoryByProjectName(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return repositories, err
|
return repositories, err
|
||||||
}
|
}
|
||||||
|
|
||||||
project := ""
|
needMatchKeyword := len(keyword) > 0 && len(keyword[0]) != 0
|
||||||
rest := ""
|
for _, repo := range repos {
|
||||||
for _, repository := range list {
|
if needMatchKeyword &&
|
||||||
project, rest = utils.ParseRepository(repository)
|
strings.Contains(repo.Name, keyword[0]) {
|
||||||
if project != name {
|
repositories = append(repositories, repo.Name)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(keyword) > 0 && len(keyword[0]) != 0 &&
|
repositories = append(repositories, repo.Name)
|
||||||
!strings.Contains(rest, keyword[0]) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
repositories = append(repositories, repository)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return repositories, nil
|
return repositories, nil
|
||||||
@ -436,3 +493,14 @@ func getReposByProject(name string, keyword ...string) ([]string, error) {
|
|||||||
func getAllRepos() ([]string, error) {
|
func getAllRepos() ([]string, error) {
|
||||||
return cache.GetRepoFromCache()
|
return cache.GetRepoFromCache()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func repositoryExist(name string, client *registry.Repository) (bool, error) {
|
||||||
|
tags, err := client.ListTag()
|
||||||
|
if err != nil {
|
||||||
|
if regErr, ok := err.(*registry_error.Error); ok && regErr.StatusCode == http.StatusNotFound {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return len(tags) != 0, nil
|
||||||
|
}
|
||||||
|
@ -157,52 +157,6 @@ func GetRecentLogs(userID, linesNum int, startTime, endTime string) ([]models.Ac
|
|||||||
return recentLogList, nil
|
return recentLogList, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
//GetTopRepos return top accessed public repos
|
|
||||||
func GetTopRepos(countNum int) ([]models.TopRepo, error) {
|
|
||||||
|
|
||||||
o := GetOrmer()
|
|
||||||
// hide the where condition: project.public = 1, Can add to the sql when necessary.
|
|
||||||
sql := "select repo_name, COUNT(repo_name) as access_count from access_log left join project on access_log.project_id=project.project_id where access_log.operation = 'pull' group by repo_name order by access_count desc limit ? "
|
|
||||||
queryParam := []interface{}{}
|
|
||||||
queryParam = append(queryParam, countNum)
|
|
||||||
var list []models.TopRepo
|
|
||||||
_, err := o.Raw(sql, queryParam).QueryRows(&list)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if len(list) == 0 {
|
|
||||||
return list, nil
|
|
||||||
}
|
|
||||||
placeHolder := make([]string, len(list))
|
|
||||||
repos := make([]string, len(list))
|
|
||||||
for i, v := range list {
|
|
||||||
repos[i] = v.RepoName
|
|
||||||
placeHolder[i] = "?"
|
|
||||||
}
|
|
||||||
placeHolderStr := strings.Join(placeHolder, ",")
|
|
||||||
queryParam = nil
|
|
||||||
queryParam = append(queryParam, repos)
|
|
||||||
var usrnameList []models.TopRepo
|
|
||||||
sql = `select a.username as creator, a.repo_name from (select access_log.repo_name, user.username,
|
|
||||||
access_log.op_time from user left join access_log on user.user_id = access_log.user_id where
|
|
||||||
access_log.operation = 'push' and access_log.repo_name in (######) order by access_log.repo_name,
|
|
||||||
access_log.op_time ASC) a group by a.repo_name`
|
|
||||||
sql = strings.Replace(sql, "######", placeHolderStr, 1)
|
|
||||||
_, err = o.Raw(sql, queryParam).QueryRows(&usrnameList)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for i := 0; i < len(list); i++ {
|
|
||||||
for _, v := range usrnameList {
|
|
||||||
if v.RepoName == list[i].RepoName {
|
|
||||||
// list[i].Creator = v.Creator
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return list, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetAccessLogCreator ...
|
// GetAccessLogCreator ...
|
||||||
func GetAccessLogCreator(repoName string) (string, error) {
|
func GetAccessLogCreator(repoName string) (string, error) {
|
||||||
o := GetOrmer()
|
o := GetOrmer()
|
||||||
|
@ -883,56 +883,10 @@ func TestGetRecentLogs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestGetTopRepos(t *testing.T) {
|
func TestGetTopRepos(t *testing.T) {
|
||||||
|
_, err := GetTopRepos(10)
|
||||||
err := ToggleProjectPublicity(currentProject.ProjectID, publicityOn)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("Error occurred in ToggleProjectPublicity: %v", err)
|
t.Fatalf("error occured in getting top repos, error: %v", err)
|
||||||
}
|
}
|
||||||
err = AccessLog(currentUser.Username, currentProject.Name, currentProject.Name+"/ubuntu", repoTag2, "push")
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("Error occurred in AccessLog: %v", err)
|
|
||||||
}
|
|
||||||
err = AccessLog(currentUser.Username, currentProject.Name, currentProject.Name+"/ubuntu", repoTag2, "pull")
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("Error occurred in AccessLog: %v", err)
|
|
||||||
}
|
|
||||||
err = AccessLog(currentUser.Username, currentProject.Name, currentProject.Name+"/ubuntu", repoTag2, "pull")
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("Error occurred in AccessLog: %v", err)
|
|
||||||
}
|
|
||||||
err = AccessLog(currentUser.Username, currentProject.Name, currentProject.Name+"/ubuntu", repoTag2, "pull")
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("Error occurred in AccessLog: %v", err)
|
|
||||||
}
|
|
||||||
err = AccessLog(currentUser.Username, currentProject.Name, currentProject.Name+"/ubuntu", repoTag2, "pull")
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("Error occurred in AccessLog: %v", err)
|
|
||||||
}
|
|
||||||
topRepos, err := GetTopRepos(10)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("error occured in getting top repos, error: %v", err)
|
|
||||||
}
|
|
||||||
if topRepos[0].RepoName != currentProject.Name+"/ubuntu" {
|
|
||||||
t.Errorf("error occured in get top reop's name, expected: %v, actual: %v", currentProject.Name+"/ubuntu", topRepos[0].RepoName)
|
|
||||||
}
|
|
||||||
if topRepos[0].AccessCount != 4 {
|
|
||||||
t.Errorf("error occured in get top reop's access count, expected: %v, actual: %v", 1, topRepos[0].AccessCount)
|
|
||||||
}
|
|
||||||
/*
|
|
||||||
if topRepos[0].Creator != currentUser.Username {
|
|
||||||
t.Errorf("error occured in get top reop's creator, expected: %v, actual: %v", currentUser.Username, topRepos[0].Creator)
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
err = ToggleProjectPublicity(currentProject.ProjectID, publicityOff)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("Error occurred in ToggleProjectPublicity: %v", err)
|
|
||||||
}
|
|
||||||
o := GetOrmer()
|
|
||||||
_, err = o.QueryTable("access_log").Filter("operation__in", "push,pull").Delete()
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("error occurred in deleting access logs, %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDeleteUser(t *testing.T) {
|
func TestDeleteUser(t *testing.T) {
|
||||||
|
@ -184,8 +184,7 @@ func SearchProjects(userID int) ([]models.Project, error) {
|
|||||||
|
|
||||||
//GetTotalOfUserRelevantProjects returns the total count of
|
//GetTotalOfUserRelevantProjects returns the total count of
|
||||||
// user relevant projects
|
// user relevant projects
|
||||||
func GetTotalOfUserRelevantProjects(userID int, projectName string,
|
func GetTotalOfUserRelevantProjects(userID int, projectName string) (int64, error) {
|
||||||
public ...int) (int64, error) {
|
|
||||||
o := GetOrmer()
|
o := GetOrmer()
|
||||||
sql := `select count(*) from project p
|
sql := `select count(*) from project p
|
||||||
left join project_member pm
|
left join project_member pm
|
||||||
|
@ -84,3 +84,84 @@ func RepositoryExists(name string) bool {
|
|||||||
o := GetOrmer()
|
o := GetOrmer()
|
||||||
return o.QueryTable("repository").Filter("name", name).Exist()
|
return o.QueryTable("repository").Filter("name", name).Exist()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetRepositoryByProjectName ...
|
||||||
|
func GetRepositoryByProjectName(name string) ([]*models.RepoRecord, error) {
|
||||||
|
sql := `select * from repository
|
||||||
|
where project_id = (
|
||||||
|
select project_id from project
|
||||||
|
where name = ?
|
||||||
|
)`
|
||||||
|
repos := []*models.RepoRecord{}
|
||||||
|
_, err := GetOrmer().Raw(sql, name).QueryRows(&repos)
|
||||||
|
return repos, err
|
||||||
|
}
|
||||||
|
|
||||||
|
//GetTopRepos returns the most popular repositories
|
||||||
|
func GetTopRepos(count int) ([]models.TopRepo, error) {
|
||||||
|
topRepos := []models.TopRepo{}
|
||||||
|
|
||||||
|
repositories := []*models.RepoRecord{}
|
||||||
|
if _, err := GetOrmer().QueryTable(&models.RepoRecord{}).
|
||||||
|
OrderBy("-PullCount", "Name").Limit(count).All(&repositories); err != nil {
|
||||||
|
return topRepos, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, repository := range repositories {
|
||||||
|
topRepos = append(topRepos, models.TopRepo{
|
||||||
|
RepoName: repository.Name,
|
||||||
|
AccessCount: repository.PullCount,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return topRepos, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTotalOfRepositories ...
|
||||||
|
func GetTotalOfRepositories(name string) (int64, error) {
|
||||||
|
qs := GetOrmer().QueryTable(&models.RepoRecord{})
|
||||||
|
if len(name) != 0 {
|
||||||
|
qs = qs.Filter("Name__contains", name)
|
||||||
|
}
|
||||||
|
return qs.Count()
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTotalOfPublicRepositories ...
|
||||||
|
func GetTotalOfPublicRepositories(name string) (int64, error) {
|
||||||
|
params := []interface{}{}
|
||||||
|
sql := `select count(*) from repository r
|
||||||
|
join project p
|
||||||
|
on r.project_id = p.project_id and p.public = 1 `
|
||||||
|
if len(name) != 0 {
|
||||||
|
sql += ` where r.name like ?`
|
||||||
|
params = append(params, "%"+name+"%")
|
||||||
|
}
|
||||||
|
|
||||||
|
var total int64
|
||||||
|
err := GetOrmer().Raw(sql, params).QueryRow(&total)
|
||||||
|
return total, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTotalOfUserRelevantRepositories ...
|
||||||
|
func GetTotalOfUserRelevantRepositories(userID int, name string) (int64, error) {
|
||||||
|
params := []interface{}{}
|
||||||
|
sql := `select count(*)
|
||||||
|
from repository r
|
||||||
|
join (
|
||||||
|
select p.project_id, p.public
|
||||||
|
from project p
|
||||||
|
join project_member pm
|
||||||
|
on p.project_id = pm.project_id
|
||||||
|
where pm.user_id = ?
|
||||||
|
) as pp
|
||||||
|
on r.project_id = pp.project_id `
|
||||||
|
params = append(params, userID)
|
||||||
|
if len(name) != 0 {
|
||||||
|
sql += ` where r.name like ?`
|
||||||
|
params = append(params, "%"+name+"%")
|
||||||
|
}
|
||||||
|
|
||||||
|
var total int64
|
||||||
|
err := GetOrmer().Raw(sql, params).QueryRow(&total)
|
||||||
|
return total, err
|
||||||
|
}
|
||||||
|
169
dao/repository_test.go
Normal file
169
dao/repository_test.go
Normal file
@ -0,0 +1,169 @@
|
|||||||
|
/*
|
||||||
|
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package dao
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/vmware/harbor/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
project = "library"
|
||||||
|
name = "library/repository-test"
|
||||||
|
repository = &models.RepoRecord{
|
||||||
|
Name: name,
|
||||||
|
OwnerName: "admin",
|
||||||
|
ProjectName: project,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGetRepositoryByProjectName(t *testing.T) {
|
||||||
|
if err := addRepository(repository); err != nil {
|
||||||
|
t.Fatalf("failed to add repository %s: %v", name, err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if err := deleteRepository(name); err != nil {
|
||||||
|
t.Fatalf("failed to delete repository %s: %v", name, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
repositories, err := GetRepositoryByProjectName(project)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to get repositories of project %s: %v",
|
||||||
|
project, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(repositories) == 0 {
|
||||||
|
t.Fatal("unexpected length of repositories: 0, at least 1")
|
||||||
|
}
|
||||||
|
|
||||||
|
exist := false
|
||||||
|
for _, repo := range repositories {
|
||||||
|
if repo.Name == name {
|
||||||
|
exist = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !exist {
|
||||||
|
t.Errorf("there is no repository whose name is %s", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetTotalOfRepositories(t *testing.T) {
|
||||||
|
total, err := GetTotalOfRepositories("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to get total of repositoreis: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := addRepository(repository); err != nil {
|
||||||
|
t.Fatalf("failed to add repository %s: %v", name, err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if err := deleteRepository(name); err != nil {
|
||||||
|
t.Fatalf("failed to delete repository %s: %v", name, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
n, err := GetTotalOfRepositories("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to get total of repositoreis: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if n != total+1 {
|
||||||
|
t.Errorf("unexpected total: %d != %d", n, total+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetTotalOfPublicRepositories(t *testing.T) {
|
||||||
|
total, err := GetTotalOfPublicRepositories("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to get total of public repositoreis: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := addRepository(repository); err != nil {
|
||||||
|
t.Fatalf("failed to add repository %s: %v", name, err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if err := deleteRepository(name); err != nil {
|
||||||
|
t.Fatalf("failed to delete repository %s: %v", name, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
n, err := GetTotalOfPublicRepositories("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to get total of public repositoreis: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if n != total+1 {
|
||||||
|
t.Errorf("unexpected total: %d != %d", n, total+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetTotalOfUserRelevantRepositories(t *testing.T) {
|
||||||
|
total, err := GetTotalOfUserRelevantRepositories(1, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to get total of repositoreis for user %d: %v", 1, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := addRepository(repository); err != nil {
|
||||||
|
t.Fatalf("failed to add repository %s: %v", name, err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if err := deleteRepository(name); err != nil {
|
||||||
|
t.Fatalf("failed to delete repository %s: %v", name, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
users, err := GetUserByProject(1, models.User{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to list members of project %d: %v", 1, err)
|
||||||
|
}
|
||||||
|
exist := false
|
||||||
|
for _, user := range users {
|
||||||
|
if user.UserID == 1 {
|
||||||
|
exist = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !exist {
|
||||||
|
if err = AddProjectMember(1, 1, models.DEVELOPER); err != nil {
|
||||||
|
t.Fatalf("failed to add user %d to be member of project %d: %v", 1, 1, err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if err = DeleteProjectMember(1, 1); err != nil {
|
||||||
|
t.Fatalf("failed to delete user %d from member of project %d: %v", 1, 1, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
n, err := GetTotalOfUserRelevantRepositories(1, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to get total of public repositoreis for user %d: %v", 1, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if n != total+1 {
|
||||||
|
t.Errorf("unexpected total: %d != %d", n, total+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func addRepository(repository *models.RepoRecord) error {
|
||||||
|
return AddRepository(*repository)
|
||||||
|
}
|
||||||
|
|
||||||
|
func deleteRepository(name string) error {
|
||||||
|
return DeleteRepository(name)
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user