From d85b3514e07e83b0b9d2b9a5f0df29277189b1a0 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 15 Apr 2021 10:57:52 +0800 Subject: [PATCH] Refactor the statistics API Refactor the statistics API Signed-off-by: Wenkai Yin --- api/v2.0/legacy_swagger.yaml | 43 -------- api/v2.0/swagger.yaml | 49 +++++++++ src/core/api/harborapi_test.go | 1 - src/core/api/statistic.go | 145 --------------------------- src/core/api/statistic_test.go | 89 ---------------- src/server/v2.0/handler/handler.go | 1 + src/server/v2.0/handler/statistic.go | 126 +++++++++++++++++++++++ src/server/v2.0/route/legacy.go | 1 - 8 files changed, 176 insertions(+), 279 deletions(-) delete mode 100644 src/core/api/statistic.go delete mode 100644 src/core/api/statistic_test.go create mode 100644 src/server/v2.0/handler/statistic.go diff --git a/api/v2.0/legacy_swagger.yaml b/api/v2.0/legacy_swagger.yaml index 7276d2281..a9abcd40e 100644 --- a/api/v2.0/legacy_swagger.yaml +++ b/api/v2.0/legacy_swagger.yaml @@ -164,22 +164,6 @@ paths: description: Project or metadata does not exist. '500': description: Internal server errors. - /statistics: - get: - summary: Get projects number and repositories number relevant to the user - description: | - This endpoint is aimed to statistic all of the projects number and repositories number relevant to the logined user, also the public projects number and repositories number. If the user is admin, he can also get total projects number and total repositories number. - tags: - - Products - responses: - '200': - description: Get the projects number and repositories number relevant to the user successfully. - schema: - $ref: '#/definitions/StatisticMap' - '401': - description: User need to log in first. - '500': - description: Unexpected internal errors. /email/ping: post: summary: Test connection and authentication with email server. @@ -441,33 +425,6 @@ definitions: username: type: string description: Username relevant to a project role member. - StatisticMap: - type: object - properties: - private_project_count: - type: integer - format: int32 - description: The count of the private projects which the user is a member of. - private_repo_count: - type: integer - format: int32 - description: The count of the private repositories belonging to the projects which the user is a member of. - public_project_count: - type: integer - format: int32 - description: The count of the public projects. - public_repo_count: - type: integer - format: int32 - description: The count of the public repositories belonging to the public projects which the user is a member of. - total_project_count: - type: integer - format: int32 - description: 'The count of the total projects, only be seen when the is admin.' - total_repo_count: - type: integer - format: int32 - description: 'The count of the total repositories, only be seen when the user is admin.' LdapConf: type: object properties: diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 4a4134bf1..53b206e21 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -54,6 +54,22 @@ paths: $ref: '#/definitions/Search' '500': $ref: '#/responses/500' + /statistics: + get: + summary: Get the statistic information about the projects and repositories + description: Get the statistic information about the projects and repositories + tags: + - statistic + operationId: getStatistic + responses: + '200': + description: The statistic information + schema: + $ref: '#/definitions/Statistic' + '401': + $ref: '#/responses/401' + '500': + $ref: '#/responses/500' /ldap/ping: post: operationId: pingLdap @@ -7937,3 +7953,36 @@ definitions: error: type: string description: (optional) The error message when the status is "unhealthy" + Statistic: + type: object + properties: + private_project_count: + type: integer + format: int64 + description: The count of the private projects + x-omitempty: false + private_repo_count: + type: integer + format: int64 + description: The count of the private repositories + x-omitempty: false + public_project_count: + type: integer + format: int64 + description: The count of the public projects + x-omitempty: false + public_repo_count: + type: integer + format: int64 + description: The count of the public repositories + x-omitempty: false + total_project_count: + type: integer + format: int64 + description: The count of the total projects, only be seen by the system admin + x-omitempty: false + total_repo_count: + type: integer + format: int64 + description: The count of the total repositories, only be seen by the system admin + x-omitempty: false diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index 9670c5294..790349e90 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -96,7 +96,6 @@ func init() { beego.Router("/api/projects/:id([0-9]+)/metadatas/?:name", &MetadataAPI{}, "get:Get") beego.Router("/api/projects/:id([0-9]+)/metadatas/", &MetadataAPI{}, "post:Post") beego.Router("/api/projects/:id([0-9]+)/metadatas/:name", &MetadataAPI{}, "put:Put;delete:Delete") - beego.Router("/api/statistics", &StatisticAPI{}) beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping") // Charts are controlled under projects diff --git a/src/core/api/statistic.go b/src/core/api/statistic.go deleted file mode 100644 index ff29eb9ac..000000000 --- a/src/core/api/statistic.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "errors" - "fmt" - - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/security/local" - "github.com/goharbor/harbor/src/controller/project" - "github.com/goharbor/harbor/src/lib/log" - "github.com/goharbor/harbor/src/lib/q" -) - -const ( - // PriPC : count of private projects - PriPC = "private_project_count" - // PriRC : count of private repositories - PriRC = "private_repo_count" - // PubPC : count of public projects - PubPC = "public_project_count" - // PubRC : count of public repositories - PubRC = "public_repo_count" - // TPC : total count of projects - TPC = "total_project_count" - // TRC : total count of repositories - TRC = "total_repo_count" -) - -// StatisticAPI handles request to /api/statistics/ -type StatisticAPI struct { - BaseController - username string -} - -// Prepare validates the URL and the user -func (s *StatisticAPI) Prepare() { - s.BaseController.Prepare() - if !s.SecurityCtx.IsAuthenticated() { - s.SendUnAuthorizedError(errors.New("UnAuthorized")) - return - } - s.username = s.SecurityCtx.GetUsername() -} - -// Get total projects and repos of the user -func (s *StatisticAPI) Get() { - statistic := map[string]int64{} - pubProjs, err := s.ProjectCtl.List(s.Context(), q.New(q.KeyWords{"public": true}), project.Metadata(false)) - if err != nil { - s.ParseAndHandleError("failed to get public projects", err) - return - } - - statistic[PubPC] = (int64)(len(pubProjs)) - if len(pubProjs) == 0 { - statistic[PubRC] = 0 - } else { - ids := make([]int64, 0) - for _, p := range pubProjs { - ids = append(ids, p.ProjectID) - } - n, err := dao.GetTotalOfRepositories(&models.RepositoryQuery{ - ProjectIDs: ids, - }) - if err != nil { - log.Errorf("failed to get total of public repositories: %v", err) - s.SendInternalServerError(fmt.Errorf("failed to get total of public repositories: %v", err)) - return - } - statistic[PubRC] = n - } - - if s.SecurityCtx.IsSysAdmin() { - count, err := s.ProjectCtl.Count(s.Context(), nil) - if err != nil { - log.Errorf("failed to get total of projects: %v", err) - s.SendInternalServerError(fmt.Errorf("failed to get total of projects: %v", err)) - return - } - statistic[TPC] = count - statistic[PriPC] = count - statistic[PubPC] - - n, err := dao.GetTotalOfRepositories() - if err != nil { - log.Errorf("failed to get total of repositories: %v", err) - s.SendInternalServerError(fmt.Errorf("failed to get total of repositories: %v", err)) - return - } - statistic[TRC] = n - statistic[PriRC] = n - statistic[PubRC] - } else { - privProjectIDs := make([]int64, 0) - if sc, ok := s.SecurityCtx.(*local.SecurityContext); ok && sc.IsAuthenticated() { - user := sc.User() - member := &project.MemberQuery{ - UserID: user.UserID, - GroupIDs: user.GroupIDs, - } - - myProjects, err := s.ProjectCtl.List(s.Context(), q.New(q.KeyWords{"member": member, "public": false}), project.Metadata(false)) - if err != nil { - s.ParseAndHandleError(fmt.Sprintf( - "failed to get projects of user %s", s.username), err) - return - } - for _, p := range myProjects { - privProjectIDs = append(privProjectIDs, p.ProjectID) - } - } - - statistic[PriPC] = int64(len(privProjectIDs)) - if statistic[PriPC] == 0 { - statistic[PriRC] = 0 - } else { - n, err := dao.GetTotalOfRepositories(&models.RepositoryQuery{ - ProjectIDs: privProjectIDs, - }) - if err != nil { - s.SendInternalServerError(fmt.Errorf( - "failed to get total of repositories for user %s: %v", - s.username, err)) - return - } - statistic[PriRC] = n - } - } - - s.Data["json"] = statistic - s.ServeJSON() -} diff --git a/src/core/api/statistic_test.go b/src/core/api/statistic_test.go deleted file mode 100644 index 5c9be4229..000000000 --- a/src/core/api/statistic_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package api - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - // "github.com/goharbor/harbor/src/testing/apitests/apilib" -) - -func TestStatisticGet(t *testing.T) { - - fmt.Println("Testing Statistic API") - assert := assert.New(t) - - apiTest := newHarborAPI() - - // prepare for test - - var privateProjectCount, privateRepoCount int32 - var priPublicProjectCount, priPublicRepoCount int32 - var priTotalProjectCount, priTotalRepoCount int32 - - // case 1: case 1: user not login, expect fail to get status info. - fmt.Println("case 1: user not login, expect fail to get status info.") - httpStatusCode, result, err := apiTest.StatisticGet(*unknownUsr) - if err != nil { - t.Error("Error get statistic info.", err.Error()) - t.Log(err) - } else { - assert.Equal(httpStatusCode, int(401), "Case 1: Get status info without login. (401)") - } - - // case 2: admin successful login, expect get status info successful. - fmt.Println("case 2: admin successful login, expect get status info successful.") - httpStatusCode, result, err = apiTest.StatisticGet(*admin) - if err != nil { - t.Error("Error get statistic info.", err.Error()) - t.Log(err) - } else { - assert.Equal(httpStatusCode, int(200), "Case 2: Get status info with admin login. (200)") - // fmt.Println("pri status data %+v", result) - privateProjectCount = result.PrivateProjectCount - privateRepoCount = result.PrivateRepoCount - priPublicProjectCount = result.PublicProjectCount - priPublicRepoCount = result.PublicRepoCount - priTotalProjectCount = result.TotalProjectCount - priTotalRepoCount = result.TotalRepoCount - } - - // case 3: status info increased after add more project and repo. - fmt.Println("case 3: status info increased after add more project and repo.") - - CommonAddProject() - CommonAddRepository() - - httpStatusCode, result, err = apiTest.StatisticGet(*admin) - // fmt.Println("new status data %+v", result) - - if err != nil { - t.Error("Error while get statistic information", err.Error()) - t.Log(err) - } else { - assert.Equal(privateProjectCount+1, result.PrivateProjectCount, "PrivateProjectCount should be +1") - assert.Equal(privateRepoCount, result.PrivateRepoCount) - assert.Equal(priPublicProjectCount, result.PublicProjectCount, "PublicProjectCount should be equal") - assert.Equal(priPublicRepoCount+1, result.PublicRepoCount, "PublicRepoCount should be +1") - assert.Equal(priTotalProjectCount+1, result.TotalProjectCount, "TotalProCount should be +1") - assert.Equal(priTotalRepoCount+1, result.TotalRepoCount, "TotalRepoCount should be +1") - - } - - // delete the project and repo - CommonDelProject() - CommonDelRepository() -} diff --git a/src/server/v2.0/handler/handler.go b/src/server/v2.0/handler/handler.go index 0de493148..a80333b94 100644 --- a/src/server/v2.0/handler/handler.go +++ b/src/server/v2.0/handler/handler.go @@ -62,6 +62,7 @@ func New() http.Handler { UsergroupAPI: newUserGroupAPI(), UserAPI: newUsersAPI(), HealthAPI: newHealthAPI(), + StatisticAPI: newStatisticAPI(), }) if err != nil { log.Fatal(err) diff --git a/src/server/v2.0/handler/statistic.go b/src/server/v2.0/handler/statistic.go new file mode 100644 index 000000000..712189651 --- /dev/null +++ b/src/server/v2.0/handler/statistic.go @@ -0,0 +1,126 @@ +// Copyright 2018 Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handler + +import ( + "context" + + "github.com/go-openapi/runtime/middleware" + "github.com/goharbor/harbor/src/common/security/local" + "github.com/goharbor/harbor/src/controller/project" + "github.com/goharbor/harbor/src/controller/repository" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/server/v2.0/models" + operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/statistic" +) + +func newStatisticAPI() *statisticAPI { + return &statisticAPI{ + proCtl: project.Ctl, + repoCtl: repository.Ctl, + } +} + +type statisticAPI struct { + BaseAPI + proCtl project.Controller + repoCtl repository.Controller +} + +func (s *statisticAPI) GetStatistic(ctx context.Context, params operation.GetStatisticParams) middleware.Responder { + if err := s.RequireAuthenticated(ctx); err != nil { + return s.SendError(ctx, err) + } + + statistic := &models.Statistic{} + pubProjs, err := s.proCtl.List(ctx, q.New(q.KeyWords{"public": true}), project.Metadata(false)) + if err != nil { + return s.SendError(ctx, err) + } + + statistic.PublicProjectCount = (int64)(len(pubProjs)) + if len(pubProjs) == 0 { + statistic.PublicRepoCount = 0 + } else { + var ids []interface{} + for _, p := range pubProjs { + ids = append(ids, p.ProjectID) + } + n, err := s.repoCtl.Count(ctx, &q.Query{ + Keywords: map[string]interface{}{ + "ProjectID": q.NewOrList(ids), + }, + }) + if err != nil { + return s.SendError(ctx, err) + } + statistic.PublicRepoCount = n + } + + securityCtx, err := s.GetSecurityContext(ctx) + if err != nil { + return s.SendError(ctx, err) + } + + if securityCtx.IsSysAdmin() { + count, err := s.proCtl.Count(ctx, nil) + if err != nil { + return s.SendError(ctx, err) + } + statistic.TotalProjectCount = count + statistic.PrivateProjectCount = count - statistic.PublicProjectCount + + n, err := s.repoCtl.Count(ctx, nil) + if err != nil { + return s.SendError(ctx, err) + } + statistic.TotalRepoCount = n + statistic.PrivateRepoCount = n - statistic.PublicRepoCount + } else { + var privProjectIDs []interface{} + if sc, ok := securityCtx.(*local.SecurityContext); ok && sc.IsAuthenticated() { + user := sc.User() + member := &project.MemberQuery{ + UserID: user.UserID, + GroupIDs: user.GroupIDs, + } + + myProjects, err := s.proCtl.List(ctx, q.New(q.KeyWords{"member": member, "public": false}), project.Metadata(false)) + if err != nil { + return s.SendError(ctx, err) + } + for _, p := range myProjects { + privProjectIDs = append(privProjectIDs, p.ProjectID) + } + } + + statistic.PrivateProjectCount = int64(len(privProjectIDs)) + if statistic.PrivateProjectCount == 0 { + statistic.PrivateRepoCount = 0 + } else { + n, err := s.repoCtl.Count(ctx, &q.Query{ + Keywords: map[string]interface{}{ + "ProjectID": q.NewOrList(privProjectIDs), + }, + }) + if err != nil { + return s.SendError(ctx, err) + } + statistic.PrivateRepoCount = n + } + } + + return operation.NewGetStatisticOK().WithPayload(statistic) +} diff --git a/src/server/v2.0/route/legacy.go b/src/server/v2.0/route/legacy.go index 5548684d7..b9f540467 100755 --- a/src/server/v2.0/route/legacy.go +++ b/src/server/v2.0/route/legacy.go @@ -26,7 +26,6 @@ func registerLegacyRoutes() { beego.Router("/api/"+version+"/email/ping", &api.EmailAPI{}, "post:Ping") beego.Router("/api/"+version+"/projects/:id([0-9]+)/metadatas/?:name", &api.MetadataAPI{}, "get:Get") beego.Router("/api/"+version+"/projects/:id([0-9]+)/metadatas/", &api.MetadataAPI{}, "post:Post") - beego.Router("/api/"+version+"/statistics", &api.StatisticAPI{}) // APIs for chart repository if config.WithChartMuseum() {