From b14762ee17697c5ef353036033dc2f5eae318caf Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Sat, 7 Mar 2020 11:28:25 +0800 Subject: [PATCH] Add support for querying artifact by labels and tags Add support for querying artifact by labels and tags Signed-off-by: Wenkai Yin --- api/v2.0/swagger.yaml | 2 + src/internal/orm/query.go | 6 ++ src/pkg/artifact/dao/dao.go | 139 ++++++++++++++++++++------ src/pkg/q/query.go | 18 +--- src/pkg/q/query_test.go | 46 --------- src/server/v2.0/handler/artifact.go | 19 +--- src/server/v2.0/handler/base.go | 22 ++++ src/server/v2.0/handler/base_test.go | 16 +++ src/server/v2.0/handler/repository.go | 16 +-- 9 files changed, 165 insertions(+), 119 deletions(-) delete mode 100644 src/pkg/q/query_test.go diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 51c38e08f..412cf9c47 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -26,6 +26,7 @@ paths: parameters: - $ref: '#/parameters/requestId' - $ref: '#/parameters/projectName' + # TODO remove it - $ref: '#/parameters/page' - $ref: '#/parameters/pageSize' - $ref: '#/parameters/query' @@ -150,6 +151,7 @@ paths: - $ref: '#/parameters/repositoryName' - $ref: '#/parameters/page' - $ref: '#/parameters/pageSize' + - $ref: '#/parameters/query' - name: type in: query description: Query the artifacts by type. Valid values can be "IMAGE", "CHART", etc. diff --git a/src/internal/orm/query.go b/src/internal/orm/query.go index a3ed6e221..9d27bb19f 100644 --- a/src/internal/orm/query.go +++ b/src/internal/orm/query.go @@ -16,6 +16,7 @@ package orm import ( "context" + "github.com/goharbor/harbor/src/common/dao" "reflect" "strings" @@ -151,3 +152,8 @@ func ParamPlaceholderForIn(n int) string { } return strings.Join(placeholders, ",") } + +// Escape special characters +func Escape(str string) string { + return dao.Escape(str) +} diff --git a/src/pkg/artifact/dao/dao.go b/src/pkg/artifact/dao/dao.go index a876d087a..bce5e8a57 100644 --- a/src/pkg/artifact/dao/dao.go +++ b/src/pkg/artifact/dao/dao.go @@ -16,6 +16,8 @@ package dao import ( "context" + "fmt" + "strings" beegoorm "github.com/astaxie/beego/orm" ierror "github.com/goharbor/harbor/src/internal/error" @@ -52,7 +54,7 @@ type DAO interface { const ( // both tagged and untagged artifacts - all = `IN ( + both = `IN ( SELECT DISTINCT art.id FROM artifact art LEFT JOIN tag ON art.id=tag.artifact_id LEFT JOIN artifact_reference ref ON art.id=ref.child_id @@ -60,13 +62,12 @@ const ( // only untagged artifacts untagged = `IN ( SELECT DISTINCT art.id FROM artifact art - LEFT JOIN tag ON art.id=tag.artifact_id - LEFT JOIN artifact_reference ref ON art.id=ref.child_id - WHERE tag.id IS NULL AND ref.id IS NULL)` + LEFT JOIN tag ON art.id=tag.artifact_id + WHERE tag.id IS NULL)` // only tagged artifacts tagged = `IN ( SELECT DISTINCT art.id FROM artifact art - LEFT JOIN tag ON art.id=tag.artifact_id + JOIN tag ON art.id=tag.artifact_id WHERE tag.id IS NOT NULL)` ) @@ -251,31 +252,113 @@ func (d *dao) DeleteReferences(ctx context.Context, parentID int64) error { } func querySetter(ctx context.Context, query *q.Query) (beegoorm.QuerySeter, error) { - // as we modify the query in this method, copy it to avoid the impact for the original one - query = q.Copy(query) - // show both tagged and untagged artifacts by default - rawFilter := all - if query != nil && len(query.Keywords) > 0 { - value, ok := query.Keywords["Tags"] - if ok { - switch value.(string) { - case "nil": - // only show untagged artifacts - rawFilter = untagged - case "*": - // only show tagged artifacts - rawFilter = tagged - default: - return nil, ierror.New(nil).WithCode(ierror.BadRequestCode). - WithMessage("invalid value of tags: %s", value.(string)) - } - // as the "Tags" isn't a table column, remove the "Tags" from the query to avoid orm error - delete(query.Keywords, "Tags") - } - } qs, err := orm.QuerySetter(ctx, &Artifact{}, query) if err != nil { return nil, err } - return qs.FilterRaw("id", rawFilter), nil + qs, err = setBaseQuery(qs, query) + if err != nil { + return nil, err + } + qs, err = setTagQuery(qs, query) + if err != nil { + return nil, err + } + qs, err = setLabelQuery(qs, query) + if err != nil { + return nil, err + } + return qs, nil +} + +// handle q=base=* +// when "q=base=*" is specified in the query, the base collection is the all artifacts of database, +// otherwise the base connection is only the tagged artifacts and untagged artifacts that aren't +// referenced by others +func setBaseQuery(qs beegoorm.QuerySeter, query *q.Query) (beegoorm.QuerySeter, error) { + if query == nil || len(query.Keywords) == 0 { + qs = qs.FilterRaw("id", both) + return qs, nil + } + base, exist := query.Keywords["base"] + if !exist { + qs = qs.FilterRaw("id", both) + return qs, nil + } + b, ok := base.(string) + if !ok || b != "*" { + return qs, ierror.New(nil).WithCode(ierror.BadRequestCode). + WithMessage(`the value of "base" query can only be exact match value with "*"`) + } + // the base is specified as "*" + return qs, nil +} + +// handle query string: q=tags=value q=tags=~value +func setTagQuery(qs beegoorm.QuerySeter, query *q.Query) (beegoorm.QuerySeter, error) { + if query == nil || len(query.Keywords) == 0 { + return qs, nil + } + tags, exist := query.Keywords["tags"] + if !exist { + tags, exist = query.Keywords["Tags"] + if !exist { + return qs, nil + } + } + + // fuzzy match + f, ok := tags.(*q.FuzzyMatchValue) + if ok { + sql := fmt.Sprintf(`IN ( + SELECT DISTINCT art.id FROM artifact art + JOIN tag ON art.id=tag.artifact_id + WHERE tag.name LIKE '%%%s%%')`, orm.Escape(f.Value)) + qs = qs.FilterRaw("id", sql) + return qs, nil + } + // exact match, only handle "*" for listing tagged artifacts and "nil" for listing untagged artifacts + s, ok := tags.(string) + if ok { + if s == "*" { + qs = qs.FilterRaw("id", tagged) + return qs, nil + } + if s == "nil" { + qs = qs.FilterRaw("id", untagged) + return qs, nil + } + } + return qs, ierror.New(nil).WithCode(ierror.BadRequestCode). + WithMessage(`the value of "tags" query can only be fuzzy match value or exact match value with "*" and "nil"`) +} + +// handle query string: q=labels=(1 2 3) +func setLabelQuery(qs beegoorm.QuerySeter, query *q.Query) (beegoorm.QuerySeter, error) { + if query == nil || len(query.Keywords) == 0 { + return qs, nil + } + labels, exist := query.Keywords["labels"] + if !exist { + labels, exist = query.Keywords["Labels"] + if !exist { + return qs, nil + } + } + al, ok := labels.(*q.AndList) + if !ok { + return qs, ierror.New(nil).WithCode(ierror.BadRequestCode). + WithMessage(`the value of "labels" query can only be integer list with intersetion relationship`) + } + var collections []string + for _, value := range al.Values { + labelID, ok := value.(int64) + if !ok { + return qs, ierror.New(nil).WithCode(ierror.BadRequestCode). + WithMessage(`the value of "labels" query can only be integer list with intersetion relationship`) + } + collections = append(collections, fmt.Sprintf(`SELECT artifact_id FROM label_reference WHERE label_id=%d`, labelID)) + } + qs = qs.FilterRaw("id", fmt.Sprintf(`IN (%s)`, strings.Join(collections, " INTERSECT "))) + return qs, nil } diff --git a/src/pkg/q/query.go b/src/pkg/q/query.go index cf1b57700..9add324d0 100644 --- a/src/pkg/q/query.go +++ b/src/pkg/q/query.go @@ -32,22 +32,6 @@ func New(kw KeyWords) *Query { return &Query{Keywords: kw} } -// Copy the specified query object -func Copy(query *Query) *Query { - if query == nil { - return nil - } - q := &Query{ - PageNumber: query.PageNumber, - PageSize: query.PageSize, - Keywords: map[string]interface{}{}, - } - for key, value := range query.Keywords { - q.Keywords[key] = value - } - return q -} - // Range query type Range struct { Min interface{} @@ -66,5 +50,5 @@ type OrList struct { // FuzzyMatchValue query type FuzzyMatchValue struct { - Value interface{} + Value string } diff --git a/src/pkg/q/query_test.go b/src/pkg/q/query_test.go deleted file mode 100644 index 12a41e3f5..000000000 --- a/src/pkg/q/query_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package q - -import ( - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "testing" -) - -func TestCopy(t *testing.T) { - // nil - q := Copy(nil) - assert.Nil(t, q) - - // not nil - query := &Query{ - PageNumber: 1, - PageSize: 10, - Keywords: map[string]interface{}{ - "key": "value", - }, - } - q = Copy(query) - require.NotNil(t, q) - assert.Equal(t, int64(1), q.PageNumber) - assert.Equal(t, int64(10), q.PageSize) - assert.Equal(t, "value", q.Keywords["key"].(string)) - // changes for the copy doesn't effect the original one - q.PageSize = 20 - q.Keywords["key"] = "value2" - assert.Equal(t, int64(10), query.PageSize) - assert.Equal(t, "value", query.Keywords["key"].(string)) -} diff --git a/src/server/v2.0/handler/artifact.go b/src/server/v2.0/handler/artifact.go index f0b5dc4bf..4bcec6d99 100644 --- a/src/server/v2.0/handler/artifact.go +++ b/src/server/v2.0/handler/artifact.go @@ -32,7 +32,6 @@ import ( "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/utils" ierror "github.com/goharbor/harbor/src/internal/error" - "github.com/goharbor/harbor/src/pkg/q" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" "github.com/goharbor/harbor/src/server/v2.0/handler/assembler" "github.com/goharbor/harbor/src/server/v2.0/handler/model" @@ -65,21 +64,11 @@ func (a *artifactAPI) ListArtifacts(ctx context.Context, params operation.ListAr if err := a.RequireProjectAccess(ctx, params.ProjectName, rbac.ActionList, rbac.ResourceArtifact); err != nil { return a.SendError(ctx, err) } + // set query - query := &q.Query{ - Keywords: map[string]interface{}{}, - } - if params.Type != nil { - query.Keywords["Type"] = *(params.Type) - } - if params.Tags != nil { - query.Keywords["Tags"] = *(params.Tags) - } - if params.Page != nil { - query.PageNumber = *(params.Page) - } - if params.PageSize != nil { - query.PageSize = *(params.PageSize) + query, err := a.BuildQuery(ctx, params.Q) + if err != nil { + return a.SendError(ctx, err) } query.Keywords["RepositoryName"] = fmt.Sprintf("%s/%s", params.ProjectName, params.RepositoryName) diff --git a/src/server/v2.0/handler/base.go b/src/server/v2.0/handler/base.go index 0b50cfe12..c99ac7691 100644 --- a/src/server/v2.0/handler/base.go +++ b/src/server/v2.0/handler/base.go @@ -21,6 +21,7 @@ import ( "errors" "github.com/goharbor/harbor/src/internal" ierror "github.com/goharbor/harbor/src/internal/error" + "github.com/goharbor/harbor/src/pkg/q" "net/url" "strconv" @@ -99,6 +100,27 @@ func (b *BaseAPI) RequireProjectAccess(ctx context.Context, projectIDOrName inte return ierror.ForbiddenError(nil) } +// BuildQuery builds the query model according to the query string +func (b *BaseAPI) BuildQuery(ctx context.Context, query *string) (*q.Query, error) { + var ( + qy *q.Query + err error + ) + if query != nil { + qy, err = q.Build(*query) + if err != nil { + return nil, err + } + } + if qy == nil { + qy = &q.Query{} + } + if qy.Keywords == nil { + qy.Keywords = map[string]interface{}{} + } + return qy, nil +} + // Links return Links based on the provided pagination information func (b *BaseAPI) Links(ctx context.Context, u *url.URL, total, pageNumber, pageSize int64) internal.Links { url := *u diff --git a/src/server/v2.0/handler/base_test.go b/src/server/v2.0/handler/base_test.go index c9bba03e0..8cbef7682 100644 --- a/src/server/v2.0/handler/base_test.go +++ b/src/server/v2.0/handler/base_test.go @@ -29,6 +29,22 @@ func (b *baseHandlerTestSuite) SetupSuite() { b.base = &BaseAPI{} } +func (b *baseHandlerTestSuite) TestBuildQuery() { + // nil query string pointer + var query *string + q, err := b.base.BuildQuery(nil, query) + b.Require().Nil(err) + b.Require().NotNil(q) + b.NotNil(q.Keywords) + + // not nil query string + str := "q=a=b" + q, err = b.base.BuildQuery(nil, &str) + b.Require().Nil(err) + b.Require().NotNil(q) + b.NotNil(q.Keywords) +} + func (b *baseHandlerTestSuite) TestLinks() { // request first page, response contains only "next" link url, err := url.Parse("http://localhost/api/artifacts?page=1&page_size=1") diff --git a/src/server/v2.0/handler/repository.go b/src/server/v2.0/handler/repository.go index ccf11ac15..9cd5c83ae 100644 --- a/src/server/v2.0/handler/repository.go +++ b/src/server/v2.0/handler/repository.go @@ -54,19 +54,9 @@ func (r *repositoryAPI) ListRepositories(ctx context.Context, params operation.L } // set query - var query *q.Query - if params.Q != nil { - query, err = q.Build(*params.Q) - if err != nil { - return r.SendError(ctx, err) - } - } - - if query == nil { - query = &q.Query{Keywords: map[string]interface{}{}} - } - if query.Keywords == nil { - query.Keywords = map[string]interface{}{} + query, err := r.BuildQuery(ctx, params.Q) + if err != nil { + return r.SendError(ctx, err) } query.Keywords["ProjectID"] = project.ProjectID