From c355034c14c5a2290b7631883bca3aa133b0f6bf Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Mon, 23 Oct 2017 14:42:30 +0800 Subject: [PATCH] Add project metadata API Project metadata API can be used to integrated with project management service which can not provide all metadatas needed by Harbor. --- docs/swagger.yaml | 143 ++++++++++++++++++++++ src/common/api/base.go | 11 ++ src/ui/api/harborapi_test.go | 48 ++++++++ src/ui/api/metadata.go | 231 +++++++++++++++++++++++++++++++++++ src/ui/api/metadata_test.go | 112 +++++++++++++++++ src/ui/api/project.go | 36 +----- src/ui/promgr/promgr.go | 6 + src/ui/promgr/promgr_test.go | 5 + src/ui/router.go | 3 + 9 files changed, 563 insertions(+), 32 deletions(-) create mode 100644 src/ui/api/metadata.go create mode 100644 src/ui/api/metadata_test.go diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 29092018a..d5ed5785c 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -302,6 +302,149 @@ paths: description: User need to log in first. '500': description: Unexpected internal errors. + '/projects/{project_id}/metadatas': + get: + summary: Get project metadata. + description: | + This endpoint returns metadata of the project specified by project ID. + parameters: + - name: project_id + in: path + description: The ID of project. + required: true + type: integer + format: int64 + tags: + - Products + responses: + '200': + description: Get metadata successfully. + schema: + $ref: '#/definitions/ProjectMetadata' + '401': + description: User need to login first. + '500': + description: Internal server errors. + post: + summary: Add metadata for the project. + description: | + This endpoint is aimed to add metadata of a project. + parameters: + - name: project_id + in: path + type: integer + format: int64 + required: true + description: Selected project ID. + - name: metadata + in: body + required: true + schema: + $ref: '#/definitions/ProjectMetadata' + description: The metadata of project. + tags: + - Products + responses: + '200': + description: Add metadata successfully. + '400': + description: Invalid request. + '401': + description: User need to log in first. + '403': + description: User does not have permission to the project. + '404': + description: Project ID does not exist. + '500': + description: Internal server errors. + '/projects/{project_id}/metadatas/{meta_name}': + get: + summary: Get project metadata + description: | + This endpoint returns specified metadata of a project. + parameters: + - name: project_id + in: path + description: Project ID for filtering results. + required: true + type: integer + format: int64 + - name: meta_name + in: path + description: The name of metadat. + required: true + type: string + tags: + - Products + responses: + '200': + description: Get metadata successfully. + schema: + $ref: '#/definitions/ProjectMetadata' + '401': + description: User need to log in first. + '500': + description: Internal server errors. + put: + summary: Update metadata of a project. + description: | + This endpoint is aimed to update the metadata of a project. + parameters: + - name: project_id + in: path + type: integer + format: int64 + required: true + description: The ID of project. + - name: meta_name + in: path + description: The name of metadat. + required: true + type: string + tags: + - Products + responses: + '200': + description: Updated metadata successfully. + '400': + description: Invalid request. + '401': + description: User need to log in first. + '403': + description: User does not have permission to the project. + '404': + description: Project or metadata does not exist. + '500': + description: Internal server errors. + delete: + summary: Delete metadata of a project + description: | + This endpoint is aimed to delete metadata of a project. + parameters: + - name: project_id + in: path + description: The ID of project. + required: true + type: integer + format: int64 + - name: meta_name + in: path + description: The name of metadat. + required: true + type: string + tags: + - Products + responses: + '200': + description: Metadata is deleted successfully. + '400': + description: Invalid requst. + '403': + description: User need to log in first. + '404': + description: Project or metadata does not exist. + '500': + description: Internal server errors. '/projects/{project_id}/members/': get: summary: Return a project's relevant role members. diff --git a/src/common/api/base.go b/src/common/api/base.go index e381a3bb3..b17917fbc 100644 --- a/src/common/api/base.go +++ b/src/common/api/base.go @@ -75,6 +75,17 @@ func (b *BaseAPI) HandleBadRequest(text string) { b.RenderError(http.StatusBadRequest, text) } +// HandleConflict ... +func (b *BaseAPI) HandleConflict(text ...string) { + msg := "" + if len(text) > 0 { + msg = text[0] + } + log.Infof("conflict: %s", msg) + + b.RenderError(http.StatusConflict, msg) +} + // HandleInternalServerError ... func (b *BaseAPI) HandleInternalServerError(text string) { log.Error(text) diff --git a/src/ui/api/harborapi_test.go b/src/ui/api/harborapi_test.go index 9b082db85..ef754bad0 100644 --- a/src/ui/api/harborapi_test.go +++ b/src/ui/api/harborapi_test.go @@ -101,6 +101,9 @@ func init() { beego.Router("/api/projects/:id([0-9]+)/logs", &ProjectAPI{}, "get:Logs") beego.Router("/api/projects/:id([0-9]+)/_deletable", &ProjectAPI{}, "get:Deletable") beego.Router("/api/projects/:pid([0-9]+)/members/?:mid", &ProjectMemberAPI{}, "get:Get;post:Post;delete:Delete;put:Put") + beego.Router("/api/projects/:id([0-9]+)/metadatas/?:name", &MetadataAPI{}, "get:Get") + beego.Router("/api/projects/:id([0-9]+)/metadatas/", &MetadataAPI{}, "post:Post") + beego.Router("/api/projects/:id([0-9]+)/metadatas/:name", &MetadataAPI{}, "put:Put;delete:Delete") beego.Router("/api/repositories", &RepositoryAPI{}) beego.Router("/api/statistics", &StatisticAPI{}) beego.Router("/api/users/?:id", &UserAPI{}) @@ -1045,3 +1048,48 @@ func (a testapi) PingEmail(authInfo usrInfo, settings []byte) (int, string, erro return code, string(body), err } + +func (a testapi) PostMeta(authInfor usrInfo, projectID int64, metas map[string]string) (int, string, error) { + _sling := sling.New().Base(a.basePath). + Post(fmt.Sprintf("/api/projects/%d/metadatas/", projectID)). + BodyJSON(metas) + + code, body, err := request(_sling, jsonAcceptHeader, authInfor) + return code, string(body), err +} + +func (a testapi) PutMeta(authInfor usrInfo, projectID int64, name string, + metas map[string]string) (int, string, error) { + _sling := sling.New().Base(a.basePath). + Put(fmt.Sprintf("/api/projects/%d/metadatas/%s", projectID, name)). + BodyJSON(metas) + + code, body, err := request(_sling, jsonAcceptHeader, authInfor) + return code, string(body), err +} + +func (a testapi) GetMeta(authInfor usrInfo, projectID int64, name ...string) (int, map[string]string, error) { + _sling := sling.New().Base(a.basePath). + Get(fmt.Sprintf("/api/projects/%d/metadatas/", projectID)) + if len(name) > 0 { + _sling = _sling.Path(name[0]) + } + + code, body, err := request(_sling, jsonAcceptHeader, authInfor) + if err == nil && code == http.StatusOK { + metas := map[string]string{} + if err := json.Unmarshal(body, &metas); err != nil { + return 0, nil, err + } + return code, metas, nil + } + return code, nil, err +} + +func (a testapi) DeleteMeta(authInfor usrInfo, projectID int64, name string) (int, string, error) { + _sling := sling.New().Base(a.basePath). + Delete(fmt.Sprintf("/api/projects/%d/metadatas/%s", projectID, name)) + + code, body, err := request(_sling, jsonAcceptHeader, authInfor) + return code, string(body), err +} diff --git a/src/ui/api/metadata.go b/src/ui/api/metadata.go new file mode 100644 index 000000000..c88fe587d --- /dev/null +++ b/src/ui/api/metadata.go @@ -0,0 +1,231 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "fmt" + "net/http" + "reflect" + "strconv" + "strings" + + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/promgr/metamgr" +) + +// MetadataAPI ... +type MetadataAPI struct { + BaseController + metaMgr metamgr.ProjectMetadataManager + project *models.Project + name string +} + +// Prepare ... +func (m *MetadataAPI) Prepare() { + m.BaseController.Prepare() + + m.metaMgr = m.ProjectMgr.GetMetadataManager() + + // the project manager doesn't use a project metadata manager + if m.metaMgr == nil { + log.Debug("the project manager doesn't use a project metadata manager") + m.RenderError(http.StatusMethodNotAllowed, "") + return + } + + id, err := m.GetInt64FromPath(":id") + if err != nil || id <= 0 { + text := "invalid project ID: " + if err != nil { + text += err.Error() + } else { + text += fmt.Sprintf("%d", id) + } + m.HandleBadRequest(text) + return + } + + project, err := m.ProjectMgr.Get(id) + if err != nil { + m.ParseAndHandleError(fmt.Sprintf("failed to get project %d", id), err) + return + } + + if project == nil { + m.HandleNotFound(fmt.Sprintf("project %d not found", id)) + return + } + + m.project = project + + switch m.Ctx.Request.Method { + case http.MethodGet, http.MethodPost, http.MethodPut, http.MethodDelete: + if !(m.Ctx.Request.Method == http.MethodGet && project.IsPublic()) { + if !m.SecurityCtx.IsAuthenticated() { + m.HandleUnauthorized() + return + } + if !m.SecurityCtx.HasReadPerm(project.ProjectID) { + m.HandleForbidden(m.SecurityCtx.GetUsername()) + return + } + } + default: + log.Debugf("%s method not allowed", m.Ctx.Request.Method) + m.RenderError(http.StatusMethodNotAllowed, "") + return + } + + name := m.GetStringFromPath(":name") + if len(name) > 0 { + m.name = name + metas, err := m.metaMgr.Get(project.ProjectID, name) + if err != nil { + m.HandleInternalServerError(fmt.Sprintf("failed to get metadata of project %d: %v", project.ProjectID, err)) + return + } + if len(metas) == 0 { + m.HandleNotFound(fmt.Sprintf("metadata %s of project %d not found", name, project.ProjectID)) + return + } + } +} + +// Get ... +func (m *MetadataAPI) Get() { + var metas map[string]string + var err error + if len(m.name) > 0 { + metas, err = m.metaMgr.Get(m.project.ProjectID, m.name) + } else { + metas, err = m.metaMgr.Get(m.project.ProjectID) + } + + if err != nil { + m.HandleInternalServerError(fmt.Sprintf("failed to get metadata %s of project %d: %v", m.name, m.project.ProjectID, err)) + return + } + m.Data["json"] = metas + m.ServeJSON() +} + +// Post ... +func (m *MetadataAPI) Post() { + var metas map[string]string + m.DecodeJSONReq(&metas) + + ms, err := validateProjectMetadata(metas) + if err != nil { + m.HandleBadRequest(err.Error()) + return + } + + if len(ms) != 1 { + m.HandleBadRequest("invalid request: has no valid key/value pairs or has more than one valid key/value pairs") + return + } + + keys := reflect.ValueOf(ms).MapKeys() + mts, err := m.metaMgr.Get(m.project.ProjectID, keys[0].String()) + if err != nil { + m.HandleInternalServerError(fmt.Sprintf("failed to get metadata for project %d: %v", m.project.ProjectID, err)) + return + } + + if len(mts) != 0 { + m.HandleConflict() + return + } + + if err := m.metaMgr.Add(m.project.ProjectID, ms); err != nil { + m.HandleInternalServerError(fmt.Sprintf("failed to create metadata for project %d: %v", m.project.ProjectID, err)) + return + } + + m.Ctx.ResponseWriter.WriteHeader(http.StatusCreated) +} + +// Put ... +func (m *MetadataAPI) Put() { + var metas map[string]string + m.DecodeJSONReq(&metas) + + meta, exist := metas[m.name] + if !exist { + m.HandleBadRequest(fmt.Sprintf("must contains key %s", m.name)) + return + } + + ms, err := validateProjectMetadata(map[string]string{ + m.name: meta, + }) + if err != nil { + m.HandleBadRequest(err.Error()) + return + } + + if err := m.metaMgr.Update(m.project.ProjectID, map[string]string{ + m.name: ms[m.name], + }); err != nil { + m.HandleInternalServerError(fmt.Sprintf("failed to update metadata %s of project %d: %v", m.name, m.project.ProjectID, err)) + return + } +} + +// Delete ... +func (m *MetadataAPI) Delete() { + if err := m.metaMgr.Delete(m.project.ProjectID, m.name); err != nil { + m.HandleInternalServerError(fmt.Sprintf("failed to delete metadata %s of project %d: %v", m.name, m.project.ProjectID, err)) + return + } +} + +// validate metas and return a new map which contains the valid key/value pairs only +func validateProjectMetadata(metas map[string]string) (map[string]string, error) { + if len(metas) == 0 { + return nil, nil + } + + boolMetas := []string{ + models.ProMetaPublic, + models.ProMetaEnableContentTrust, + models.ProMetaPreventVul, + models.ProMetaAutoScan} + + for _, boolMeta := range boolMetas { + value, exist := metas[boolMeta] + if exist { + b, err := strconv.ParseBool(value) + if err != nil { + return nil, fmt.Errorf("failed to parse %s to bool: %v", value, err) + } + metas[boolMeta] = strconv.FormatBool(b) + } + } + + value, exist := metas[models.ProMetaSeverity] + if exist { + switch strings.ToLower(value) { + case models.SeverityHigh, models.SeverityMedium, models.SeverityLow, models.SeverityNone: + metas[models.ProMetaSeverity] = strings.ToLower(value) + default: + return nil, fmt.Errorf("invalid severity %s", value) + } + } + + return metas, nil +} diff --git a/src/ui/api/metadata_test.go b/src/ui/api/metadata_test.go new file mode 100644 index 000000000..e71a29527 --- /dev/null +++ b/src/ui/api/metadata_test.go @@ -0,0 +1,112 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmware/harbor/src/common/models" +) + +func TestValidateProjectMetadata(t *testing.T) { + var metas map[string]string + + // nil metas + ms, err := validateProjectMetadata(metas) + require.Nil(t, err) + require.Nil(t, ms) + + // valid key, invalid value(bool) + metas = map[string]string{ + models.ProMetaPublic: "invalid_value", + } + ms, err = validateProjectMetadata(metas) + require.NotNil(t, err) + + // valid key/value(bool) + metas = map[string]string{ + models.ProMetaPublic: "1", + } + ms, err = validateProjectMetadata(metas) + require.Nil(t, err) + assert.Equal(t, "true", ms[models.ProMetaPublic]) + + // valid key, invalid value(string) + metas = map[string]string{ + models.ProMetaSeverity: "invalid_value", + } + ms, err = validateProjectMetadata(metas) + require.NotNil(t, err) + + // valid key, valid value(string) + metas = map[string]string{ + models.ProMetaSeverity: "High", + } + ms, err = validateProjectMetadata(metas) + require.Nil(t, err) + assert.Equal(t, "high", ms[models.ProMetaSeverity]) +} + +func TestMetaAPI(t *testing.T) { + client := newHarborAPI() + + // non-exist project + code, _, err := client.PostMeta(*unknownUsr, int64(1000), nil) + require.Nil(t, err) + assert.Equal(t, http.StatusNotFound, code) + + // non-login + code, _, err = client.PostMeta(*unknownUsr, int64(1), nil) + require.Nil(t, err) + assert.Equal(t, http.StatusUnauthorized, code) + + // test post + code, _, err = client.PostMeta(*admin, int64(1), map[string]string{ + models.ProMetaAutoScan: "true", + }) + require.Nil(t, err) + assert.Equal(t, http.StatusCreated, code) + + // test get + code, metas, err := client.GetMeta(*admin, int64(1), models.ProMetaAutoScan) + require.Nil(t, err) + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, "true", metas[models.ProMetaAutoScan]) + + // test put + code, _, err = client.PutMeta(*admin, int64(1), models.ProMetaAutoScan, + map[string]string{ + models.ProMetaAutoScan: "false", + }) + require.Nil(t, err) + assert.Equal(t, http.StatusOK, code) + + code, metas, err = client.GetMeta(*admin, int64(1), models.ProMetaAutoScan) + require.Nil(t, err) + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, "false", metas[models.ProMetaAutoScan]) + + // test delete + code, _, err = client.DeleteMeta(*admin, int64(1), models.ProMetaAutoScan) + require.Nil(t, err) + assert.Equal(t, http.StatusOK, code) + + code, metas, err = client.GetMeta(*admin, int64(1), models.ProMetaAutoScan) + require.Nil(t, err) + assert.Equal(t, http.StatusNotFound, code) +} diff --git a/src/ui/api/project.go b/src/ui/api/project.go index 3a78cd2b8..c70693ec3 100644 --- a/src/ui/api/project.go +++ b/src/ui/api/project.go @@ -18,7 +18,6 @@ import ( "fmt" "net/http" "regexp" - "strings" "github.com/vmware/harbor/src/common" "github.com/vmware/harbor/src/common/dao" @@ -500,38 +499,11 @@ func validateProjectReq(req *models.ProjectRequest) error { return fmt.Errorf("project name is not in lower case or contains illegal characters") } - if req.Metadata != nil { - metas := req.Metadata - req.Metadata = map[string]string{} - - boolMetas := []string{ - models.ProMetaPublic, - models.ProMetaEnableContentTrust, - models.ProMetaPreventVul, - models.ProMetaAutoScan} - - for _, boolMeta := range boolMetas { - value, exist := metas[boolMeta] - if exist { - b, err := strconv.ParseBool(value) - if err != nil { - log.Errorf("failed to parse %s to bool: %v", value, err) - b = false - } - req.Metadata[boolMeta] = strconv.FormatBool(b) - } - } - - value, exist := metas[models.ProMetaSeverity] - if exist { - switch strings.ToLower(value) { - case models.SeverityHigh, models.SeverityMedium, models.SeverityLow, models.SeverityNone: - req.Metadata[models.ProMetaSeverity] = strings.ToLower(value) - default: - return fmt.Errorf("invalid severity %s", value) - } - } + metas, err := validateProjectMetadata(req.Metadata) + if err != nil { + return err } + req.Metadata = metas return nil } diff --git a/src/ui/promgr/promgr.go b/src/ui/promgr/promgr.go index bfa28ca15..2d3b8322e 100644 --- a/src/ui/promgr/promgr.go +++ b/src/ui/promgr/promgr.go @@ -36,6 +36,8 @@ type ProjectManager interface { Exists(projectIDOrName interface{}) (bool, error) // get all public project GetPublic() ([]*models.Project, error) + // if the project manager uses a metadata manager, return it, otherwise return nil + GetMetadataManager() metamgr.ProjectMetadataManager } type defaultProjectManager struct { @@ -235,3 +237,7 @@ func (d *defaultProjectManager) GetPublic() ([]*models.Project, error) { } return result.Projects, nil } + +func (d *defaultProjectManager) GetMetadataManager() metamgr.ProjectMetadataManager { + return d.metaMgr +} diff --git a/src/ui/promgr/promgr_test.go b/src/ui/promgr/promgr_test.go index e646217d6..061200316 100644 --- a/src/ui/promgr/promgr_test.go +++ b/src/ui/promgr/promgr_test.go @@ -119,3 +119,8 @@ func TestGetPublic(t *testing.T) { assert.Equal(t, 1, len(projects)) assert.True(t, projects[0].IsPublic()) } + +func TestGetMetadataManager(t *testing.T) { + metaMgr := proMgr.GetMetadataManager() + assert.Nil(t, metaMgr) +} diff --git a/src/ui/router.go b/src/ui/router.go index 51eca5438..a7daa550d 100644 --- a/src/ui/router.go +++ b/src/ui/router.go @@ -86,6 +86,9 @@ func initRouters() { beego.Router("/api/projects/", &api.ProjectAPI{}, "get:List;post:Post") beego.Router("/api/projects/:id([0-9]+)/logs", &api.ProjectAPI{}, "get:Logs") beego.Router("/api/projects/:id([0-9]+)/_deletable", &api.ProjectAPI{}, "get:Deletable") + beego.Router("/api/projects/:id([0-9]+)/metadatas/?:name", &api.MetadataAPI{}, "get:Get") + beego.Router("/api/projects/:id([0-9]+)/metadatas/", &api.MetadataAPI{}, "post:Post") + beego.Router("/api/projects/:id([0-9]+)/metadatas/:name", &api.MetadataAPI{}, "put:Put;delete:Delete") beego.Router("/api/internal/syncregistry", &api.InternalAPI{}, "post:SyncRegistry") beego.Router("/api/repositories", &api.RepositoryAPI{}, "get:Get") beego.Router("/api/repositories/scanAll", &api.RepositoryAPI{}, "post:ScanAll")