Merge pull request #10968 from ywk253100/200306_artifact_query

Add support for querying artifact by labels and tags

api/v2.0/swagger.yaml

@@ -26,6 +26,7 @@ paths:
       parameters:
         - $ref: '#/parameters/requestId'
         - $ref: '#/parameters/projectName'
+        # TODO remove it
         - $ref: '#/parameters/page'
         - $ref: '#/parameters/pageSize'
         - $ref: '#/parameters/query'
@@ -150,6 +151,7 @@ paths:
         - $ref: '#/parameters/repositoryName'
         - $ref: '#/parameters/page'
         - $ref: '#/parameters/pageSize'
+        - $ref: '#/parameters/query'
         - name: type
           in: query
           description: Query the artifacts by type. Valid values can be "IMAGE", "CHART", etc.

src/internal/orm/query.go

@@ -16,6 +16,7 @@ package orm
 
 import (
 	"context"
+	"github.com/goharbor/harbor/src/common/dao"
 	"reflect"
 	"strings"
 
@@ -151,3 +152,8 @@ func ParamPlaceholderForIn(n int) string {
 	}
 	return strings.Join(placeholders, ",")
 }
+
+// Escape special characters
+func Escape(str string) string {
+	return dao.Escape(str)
+}

src/pkg/artifact/dao/dao.go

@@ -16,6 +16,8 @@ package dao
 
 import (
 	"context"
+	"fmt"
+	"strings"
 
 	beegoorm "github.com/astaxie/beego/orm"
 	ierror "github.com/goharbor/harbor/src/internal/error"
@@ -52,7 +54,7 @@ type DAO interface {
 
 const (
 	// both tagged and untagged artifacts
-	all = `IN (
+	both = `IN (
 		SELECT DISTINCT art.id FROM artifact art
 		LEFT JOIN tag ON art.id=tag.artifact_id 
 		LEFT JOIN artifact_reference ref ON art.id=ref.child_id
@@ -60,13 +62,12 @@ const (
 	// only untagged artifacts
 	untagged = `IN (
 		SELECT DISTINCT art.id FROM artifact art
-		LEFT JOIN tag ON art.id=tag.artifact_id 
+		LEFT JOIN tag ON art.id=tag.artifact_id
-		LEFT JOIN artifact_reference ref ON art.id=ref.child_id
+		WHERE tag.id IS NULL)`
-		WHERE tag.id IS NULL AND ref.id IS NULL)`
 	// only tagged artifacts
 	tagged = `IN (
 		SELECT DISTINCT art.id FROM artifact art
-		LEFT JOIN tag ON art.id=tag.artifact_id
+		JOIN tag ON art.id=tag.artifact_id
 		WHERE tag.id IS NOT NULL)`
 )
 
@@ -251,31 +252,113 @@ func (d *dao) DeleteReferences(ctx context.Context, parentID int64) error {
 }
 
 func querySetter(ctx context.Context, query *q.Query) (beegoorm.QuerySeter, error) {
-	// as we modify the query in this method, copy it to avoid the impact for the original one
-	query = q.Copy(query)
-	// show both tagged and untagged artifacts by default
-	rawFilter := all
-	if query != nil && len(query.Keywords) > 0 {
-		value, ok := query.Keywords["Tags"]
-		if ok {
-			switch value.(string) {
-			case "nil":
-				// only show untagged artifacts
-				rawFilter = untagged
-			case "*":
-				// only show tagged artifacts
-				rawFilter = tagged
-			default:
-				return nil, ierror.New(nil).WithCode(ierror.BadRequestCode).
-					WithMessage("invalid value of tags: %s", value.(string))
-			}
-			// as the "Tags" isn't a table column, remove the "Tags" from the query to avoid orm error
-			delete(query.Keywords, "Tags")
-		}
-	}
 	qs, err := orm.QuerySetter(ctx, &Artifact{}, query)
 	if err != nil {
 		return nil, err
 	}
-	return qs.FilterRaw("id", rawFilter), nil
+	qs, err = setBaseQuery(qs, query)
+	if err != nil {
+		return nil, err
+	}
+	qs, err = setTagQuery(qs, query)
+	if err != nil {
+		return nil, err
+	}
+	qs, err = setLabelQuery(qs, query)
+	if err != nil {
+		return nil, err
+	}
+	return qs, nil
+}
+
+// handle q=base=*
+// when "q=base=*" is specified in the query, the base collection is the all artifacts of database,
+// otherwise the base connection is only the tagged artifacts and untagged artifacts that aren't
+// referenced by others
+func setBaseQuery(qs beegoorm.QuerySeter, query *q.Query) (beegoorm.QuerySeter, error) {
+	if query == nil || len(query.Keywords) == 0 {
+		qs = qs.FilterRaw("id", both)
+		return qs, nil
+	}
+	base, exist := query.Keywords["base"]
+	if !exist {
+		qs = qs.FilterRaw("id", both)
+		return qs, nil
+	}
+	b, ok := base.(string)
+	if !ok || b != "*" {
+		return qs, ierror.New(nil).WithCode(ierror.BadRequestCode).
+			WithMessage(`the value of "base" query can only be exact match value with "*"`)
+	}
+	// the base is specified as "*"
+	return qs, nil
+}
+
+// handle query string: q=tags=value q=tags=~value
+func setTagQuery(qs beegoorm.QuerySeter, query *q.Query) (beegoorm.QuerySeter, error) {
+	if query == nil || len(query.Keywords) == 0 {
+		return qs, nil
+	}
+	tags, exist := query.Keywords["tags"]
+	if !exist {
+		tags, exist = query.Keywords["Tags"]
+		if !exist {
+			return qs, nil
+		}
+	}
+
+	// fuzzy match
+	f, ok := tags.(*q.FuzzyMatchValue)
+	if ok {
+		sql := fmt.Sprintf(`IN (
+		SELECT DISTINCT art.id FROM artifact art
+		JOIN tag ON art.id=tag.artifact_id
+		WHERE tag.name LIKE '%%%s%%')`, orm.Escape(f.Value))
+		qs = qs.FilterRaw("id", sql)
+		return qs, nil
+	}
+	// exact match, only handle "*" for listing tagged artifacts and "nil" for listing untagged artifacts
+	s, ok := tags.(string)
+	if ok {
+		if s == "*" {
+			qs = qs.FilterRaw("id", tagged)
+			return qs, nil
+		}
+		if s == "nil" {
+			qs = qs.FilterRaw("id", untagged)
+			return qs, nil
+		}
+	}
+	return qs, ierror.New(nil).WithCode(ierror.BadRequestCode).
+		WithMessage(`the value of "tags" query can only be fuzzy match value or exact match value with "*" and "nil"`)
+}
+
+// handle query string: q=labels=(1 2 3)
+func setLabelQuery(qs beegoorm.QuerySeter, query *q.Query) (beegoorm.QuerySeter, error) {
+	if query == nil || len(query.Keywords) == 0 {
+		return qs, nil
+	}
+	labels, exist := query.Keywords["labels"]
+	if !exist {
+		labels, exist = query.Keywords["Labels"]
+		if !exist {
+			return qs, nil
+		}
+	}
+	al, ok := labels.(*q.AndList)
+	if !ok {
+		return qs, ierror.New(nil).WithCode(ierror.BadRequestCode).
+			WithMessage(`the value of "labels" query can only be integer list with intersetion relationship`)
+	}
+	var collections []string
+	for _, value := range al.Values {
+		labelID, ok := value.(int64)
+		if !ok {
+			return qs, ierror.New(nil).WithCode(ierror.BadRequestCode).
+				WithMessage(`the value of "labels" query can only be integer list with intersetion relationship`)
+		}
+		collections = append(collections, fmt.Sprintf(`SELECT artifact_id FROM label_reference WHERE label_id=%d`, labelID))
+	}
+	qs = qs.FilterRaw("id", fmt.Sprintf(`IN (%s)`, strings.Join(collections, " INTERSECT ")))
+	return qs, nil
 }

src/pkg/q/query.go

@@ -32,22 +32,6 @@ func New(kw KeyWords) *Query {
 	return &Query{Keywords: kw}
 }
 
-// Copy the specified query object
-func Copy(query *Query) *Query {
-	if query == nil {
-		return nil
-	}
-	q := &Query{
-		PageNumber: query.PageNumber,
-		PageSize:   query.PageSize,
-		Keywords:   map[string]interface{}{},
-	}
-	for key, value := range query.Keywords {
-		q.Keywords[key] = value
-	}
-	return q
-}
-
 // Range query
 type Range struct {
 	Min interface{}
@@ -66,5 +50,5 @@ type OrList struct {
 
 // FuzzyMatchValue query
 type FuzzyMatchValue struct {
-	Value interface{}
+	Value string
 }

src/pkg/q/query_test.go

@@ -1,46 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package q
-
-import (
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"testing"
-)
-
-func TestCopy(t *testing.T) {
-	// nil
-	q := Copy(nil)
-	assert.Nil(t, q)
-
-	// not nil
-	query := &Query{
-		PageNumber: 1,
-		PageSize:   10,
-		Keywords: map[string]interface{}{
-			"key": "value",
-		},
-	}
-	q = Copy(query)
-	require.NotNil(t, q)
-	assert.Equal(t, int64(1), q.PageNumber)
-	assert.Equal(t, int64(10), q.PageSize)
-	assert.Equal(t, "value", q.Keywords["key"].(string))
-	// changes for the copy doesn't effect the original one
-	q.PageSize = 20
-	q.Keywords["key"] = "value2"
-	assert.Equal(t, int64(10), query.PageSize)
-	assert.Equal(t, "value", query.Keywords["key"].(string))
-}

src/server/v2.0/handler/artifact.go

@@ -32,7 +32,6 @@ import (
 	"github.com/goharbor/harbor/src/common/rbac"
 	"github.com/goharbor/harbor/src/common/utils"
 	ierror "github.com/goharbor/harbor/src/internal/error"
-	"github.com/goharbor/harbor/src/pkg/q"
 	v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
 	"github.com/goharbor/harbor/src/server/v2.0/handler/assembler"
 	"github.com/goharbor/harbor/src/server/v2.0/handler/model"
@@ -65,21 +64,11 @@ func (a *artifactAPI) ListArtifacts(ctx context.Context, params operation.ListAr
 	if err := a.RequireProjectAccess(ctx, params.ProjectName, rbac.ActionList, rbac.ResourceArtifact); err != nil {
 		return a.SendError(ctx, err)
 	}
+
 	// set query
-	query := &q.Query{
+	query, err := a.BuildQuery(ctx, params.Q)
-		Keywords: map[string]interface{}{},
+	if err != nil {
-	}
+		return a.SendError(ctx, err)
-	if params.Type != nil {
-		query.Keywords["Type"] = *(params.Type)
-	}
-	if params.Tags != nil {
-		query.Keywords["Tags"] = *(params.Tags)
-	}
-	if params.Page != nil {
-		query.PageNumber = *(params.Page)
-	}
-	if params.PageSize != nil {
-		query.PageSize = *(params.PageSize)
 	}
 	query.Keywords["RepositoryName"] = fmt.Sprintf("%s/%s", params.ProjectName, params.RepositoryName)
 

src/server/v2.0/handler/base.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"github.com/goharbor/harbor/src/internal"
 	ierror "github.com/goharbor/harbor/src/internal/error"
+	"github.com/goharbor/harbor/src/pkg/q"
 	"net/url"
 	"strconv"
 
@@ -99,6 +100,27 @@ func (b *BaseAPI) RequireProjectAccess(ctx context.Context, projectIDOrName inte
 	return ierror.ForbiddenError(nil)
 }
 
+// BuildQuery builds the query model according to the query string
+func (b *BaseAPI) BuildQuery(ctx context.Context, query *string) (*q.Query, error) {
+	var (
+		qy  *q.Query
+		err error
+	)
+	if query != nil {
+		qy, err = q.Build(*query)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if qy == nil {
+		qy = &q.Query{}
+	}
+	if qy.Keywords == nil {
+		qy.Keywords = map[string]interface{}{}
+	}
+	return qy, nil
+}
+
 // Links return Links based on the provided pagination information
 func (b *BaseAPI) Links(ctx context.Context, u *url.URL, total, pageNumber, pageSize int64) internal.Links {
 	url := *u

src/server/v2.0/handler/base_test.go

@@ -29,6 +29,22 @@ func (b *baseHandlerTestSuite) SetupSuite() {
 	b.base = &BaseAPI{}
 }
 
+func (b *baseHandlerTestSuite) TestBuildQuery() {
+	// nil query string pointer
+	var query *string
+	q, err := b.base.BuildQuery(nil, query)
+	b.Require().Nil(err)
+	b.Require().NotNil(q)
+	b.NotNil(q.Keywords)
+
+	// not nil query string
+	str := "q=a=b"
+	q, err = b.base.BuildQuery(nil, &str)
+	b.Require().Nil(err)
+	b.Require().NotNil(q)
+	b.NotNil(q.Keywords)
+}
+
 func (b *baseHandlerTestSuite) TestLinks() {
 	// request first page, response contains only "next" link
 	url, err := url.Parse("http://localhost/api/artifacts?page=1&page_size=1")

src/server/v2.0/handler/repository.go

@@ -54,19 +54,9 @@ func (r *repositoryAPI) ListRepositories(ctx context.Context, params operation.L
 	}
 
 	// set query
-	var query *q.Query
+	query, err := r.BuildQuery(ctx, params.Q)
-	if params.Q != nil {
+	if err != nil {
-		query, err = q.Build(*params.Q)
+		return r.SendError(ctx, err)
-		if err != nil {
-			return r.SendError(ctx, err)
-		}
-	}
-
-	if query == nil {
-		query = &q.Query{Keywords: map[string]interface{}{}}
-	}
-	if query.Keywords == nil {
-		query.Keywords = map[string]interface{}{}
 	}
 	query.Keywords["ProjectID"] = project.ProjectID