From 8abb630b4ca89011193547e30e11b3abe135e197 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 4 Mar 2020 22:10:21 +0800 Subject: [PATCH] Implement query string builder This commit defines the API query string format and provides the builders to build query string to query model Signed-off-by: Wenkai Yin --- api/v2.0/swagger.yaml | 8 + src/internal/orm/query.go | 102 +++++++++- src/internal/orm/query_test.go | 61 ++++++ src/pkg/blob/dao/dao.go | 7 +- src/pkg/blob/dao/dao_test.go | 8 +- src/pkg/blob/manager.go | 6 +- src/pkg/q/builder.go | 193 ++++++++++++++++++ src/pkg/q/builder_test.go | 273 ++++++++++++++++++++++++++ src/pkg/q/query.go | 21 ++ src/server/v2.0/handler/repository.go | 26 +-- 10 files changed, 686 insertions(+), 19 deletions(-) create mode 100644 src/internal/orm/query_test.go create mode 100644 src/pkg/q/builder.go create mode 100644 src/pkg/q/builder_test.go diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 2557e7c3d..e3ab2dbfc 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -28,6 +28,8 @@ paths: - $ref: '#/parameters/projectName' - $ref: '#/parameters/page' - $ref: '#/parameters/pageSize' + - $ref: '#/parameters/query' + # TODO remove it - name: name in: query description: Query the repositories by name @@ -500,6 +502,12 @@ paths: '500': $ref: '#/responses/500' parameters: + query: + name: q + description: Query string to query resources. Supported query patterns are "exact match(k=v)", "fuzzy match(k=~v)", "range(k=[min~max])", "list with union releationship(k={v1 v2 v3})" and "list with intersetion relationship(k=(v1 v2 v3))". All of these query patterns should be put in the query string "q=xxx" and splitted by ",". e.g. q=k1=v1,k2=~v2,k3=[min~max] + in: query + type: string + required: false requestId: name: X-Request-Id description: An unique ID for the request diff --git a/src/internal/orm/query.go b/src/internal/orm/query.go index 3bcf79721..a3ed6e221 100644 --- a/src/internal/orm/query.go +++ b/src/internal/orm/query.go @@ -16,14 +16,16 @@ package orm import ( "context" + "reflect" "strings" "github.com/astaxie/beego/orm" "github.com/goharbor/harbor/src/pkg/q" ) -// QuerySetter generates the query setter according to the query -func QuerySetter(ctx context.Context, model interface{}, query *q.Query) (orm.QuerySeter, error) { +// QuerySetter generates the query setter according to the query. "ignoredCols" is used to set the +// columns that will not be queried +func QuerySetter(ctx context.Context, model interface{}, query *q.Query, ignoredCols ...string) (orm.QuerySeter, error) { ormer, err := FromContext(ctx) if err != nil { return nil, err @@ -32,7 +34,49 @@ func QuerySetter(ctx context.Context, model interface{}, query *q.Query) (orm.Qu if query == nil { return qs, nil } + + // the program will panic when querying the columns that doesn't exist + // list the supported columns first to avoid the panic + cols := listQueriableCols(model, ignoredCols...) for k, v := range query.Keywords { + if _, exist := cols[k]; !exist { + continue + } + + // fuzzy match + f, ok := v.(*q.FuzzyMatchValue) + if ok { + qs = qs.Filter(k+"__icontains", f.Value) + continue + } + + // range + r, ok := v.(*q.Range) + if ok { + if r.Min != nil { + qs = qs.Filter(k+"__gte", r.Min) + } + if r.Max != nil { + qs = qs.Filter(k+"__lte", r.Max) + } + continue + } + + // or list + ol, ok := v.(*q.OrList) + if ok { + qs = qs.Filter(k+"__in", ol.Values...) + continue + } + + // and list + _, ok = v.(*q.OrList) + if ok { + // do nothing as and list needs to be handled by the logic of DAO + continue + } + + // exact match qs = qs.Filter(k, v) } if query.PageSize > 0 { @@ -44,6 +88,60 @@ func QuerySetter(ctx context.Context, model interface{}, query *q.Query) (orm.Qu return qs, nil } +// list the columns that can be queried +// e.g. for the following model the columns that can be queried are: +// "Field2", "customized_field2", "Field3" and "field3" +// type model struct{ +// Field1 string `orm:"-"` +// Field2 string `orm:"column(customized_field2)"` +// Field3 string +// } +// +// set "ignoredCols" to ignore the specified columns +func listQueriableCols(model interface{}, ignoredCols ...string) map[string]struct{} { + if model == nil { + return nil + } + ignored := map[string]struct{}{} + for _, ig := range ignoredCols { + ignored[ig] = struct{}{} + } + cols := map[string]struct{}{} + t := reflect.Indirect(reflect.ValueOf(model)).Type() + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + orm := field.Tag.Get("orm") + if orm == "-" { + continue + } + colName := "" + for _, str := range strings.Split(orm, ";") { + if strings.HasPrefix(str, "column") { + str = strings.TrimPrefix(str, "column(") + str = strings.TrimSuffix(str, ")") + if len(str) > 0 { + colName = str + break + } + } + } + if len(colName) == 0 { + // TODO convert the field.Name to snake_case + } + if _, exist := ignored[colName]; exist { + continue + } + if _, exist := ignored[field.Name]; exist { + continue + } + if len(colName) != 0 { + cols[colName] = struct{}{} + } + cols[field.Name] = struct{}{} + } + return cols +} + // ParamPlaceholderForIn returns a string that contains placeholders for sql keyword "in" // e.g. n=3, returns "?,?,?" func ParamPlaceholderForIn(n int) string { diff --git a/src/internal/orm/query_test.go b/src/internal/orm/query_test.go new file mode 100644 index 000000000..afd3cb26c --- /dev/null +++ b/src/internal/orm/query_test.go @@ -0,0 +1,61 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package orm + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" +) + +func TestListQueriableCols(t *testing.T) { + type model struct { + Field1 string `orm:"column(field1)" json:"field1"` + Field2 string `orm:"column(customized_field2)"` + Field3 string + Field4 string `orm:"column(field4)"` + } + // without ignoring columns + cols := listQueriableCols(&model{}) + require.Len(t, cols, 7) + _, exist := cols["Field1"] + assert.True(t, exist) + _, exist = cols["field1"] + assert.True(t, exist) + _, exist = cols["Field2"] + assert.True(t, exist) + _, exist = cols["customized_field2"] + assert.True(t, exist) + _, exist = cols["Field3"] + assert.True(t, exist) + _, exist = cols["Field4"] + assert.True(t, exist) + _, exist = cols["field4"] + assert.True(t, exist) + + // with ignoring columns + cols = listQueriableCols(&model{}, "Field4") + require.Len(t, cols, 5) + _, exist = cols["Field1"] + assert.True(t, exist) + _, exist = cols["field1"] + assert.True(t, exist) + _, exist = cols["Field2"] + assert.True(t, exist) + _, exist = cols["customized_field2"] + assert.True(t, exist) + _, exist = cols["Field3"] + assert.True(t, exist) +} diff --git a/src/pkg/blob/dao/dao.go b/src/pkg/blob/dao/dao.go index 2a8321fde..3023272ac 100644 --- a/src/pkg/blob/dao/dao.go +++ b/src/pkg/blob/dao/dao.go @@ -253,8 +253,11 @@ func (d *dao) DeleteProjectBlob(ctx context.Context, projectID int64, blobIDs .. if len(blobIDs) == 0 { return nil } - - kw := q.KeyWords{"blob_id__in": blobIDs} + ol := &q.OrList{} + for _, blobID := range blobIDs { + ol.Values = append(ol.Values, blobID) + } + kw := q.KeyWords{"blob_id": ol} qs, err := orm.QuerySetter(ctx, &models.ProjectBlob{}, q.New(kw)) if err != nil { return err diff --git a/src/pkg/blob/dao/dao_test.go b/src/pkg/blob/dao/dao_test.go index 0a0f5f15f..1e37d3a2b 100644 --- a/src/pkg/blob/dao/dao_test.go +++ b/src/pkg/blob/dao/dao_test.go @@ -160,7 +160,7 @@ func (suite *DaoTestSuite) TestListBlobs() { suite.Len(blobs, 1) } - blobs, err = suite.dao.ListBlobs(ctx, q.New(q.KeyWords{"digest__in": []string{digest1, digest2}})) + blobs, err = suite.dao.ListBlobs(ctx, q.New(q.KeyWords{"digest": &q.OrList{Values: []interface{}{digest1, digest2}}})) if suite.Nil(err) { suite.Len(blobs, 2) } @@ -193,7 +193,11 @@ func (suite *DaoTestSuite) TestFindBlobsShouldUnassociatedWithProject() { } } - blobs, err := suite.dao.ListBlobs(ctx, q.New(q.KeyWords{"digest__in": blobDigests})) + ol := &q.OrList{} + for _, blobDigest := range blobDigests { + ol.Values = append(ol.Values, blobDigest) + } + blobs, err := suite.dao.ListBlobs(ctx, q.New(q.KeyWords{"digest": ol})) suite.Nil(err) suite.Len(blobs, 5) diff --git a/src/pkg/blob/manager.go b/src/pkg/blob/manager.go index 817735337..07f93e0eb 100644 --- a/src/pkg/blob/manager.go +++ b/src/pkg/blob/manager.go @@ -126,7 +126,11 @@ func (m *manager) List(ctx context.Context, params ListParams) ([]*Blob, error) } if len(params.BlobDigests) > 0 { - kw["digest__in"] = params.BlobDigests + ol := &q.OrList{} + for _, blobDigest := range params.BlobDigests { + ol.Values = append(ol.Values, blobDigest) + } + kw["digest"] = ol } blobs, err := m.dao.ListBlobs(ctx, q.New(kw)) diff --git a/src/pkg/q/builder.go b/src/pkg/q/builder.go new file mode 100644 index 000000000..59878f903 --- /dev/null +++ b/src/pkg/q/builder.go @@ -0,0 +1,193 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package q + +import ( + "fmt" + ierror "github.com/goharbor/harbor/src/internal/error" + "strconv" + "strings" + "time" +) + +// Build query sting into the Query model +// query string format: q=k=v,k=~v,k=[min~max],k={v1 v2 v3},k=(v1 v2 v3),page=1,page_size=10 +// exact match: k=v +// fuzzy match: k=~v +// range: k=[min~max] +// or list: k={v1 v2 v3} +// and list: k=(v1 v2 v3) +func Build(q string) (*Query, error) { + if len(q) == 0 { + return nil, nil + } + query := &Query{Keywords: map[string]interface{}{}} + params := strings.Split(q, ",") + for _, param := range params { + strs := strings.SplitN(param, "=", 2) + if len(strs) != 2 || len(strs[0]) == 0 || len(strs[1]) == 0 { + return nil, ierror.New(nil). + WithCode(ierror.BadRequestCode). + WithMessage(`the query string must contain "=" and the key/value cannot be empty`) + } + if strs[0] == "page" { + i, err := strconv.ParseInt(strs[1], 10, 64) + if err != nil { + return nil, ierror.New(nil). + WithCode(ierror.BadRequestCode). + WithMessage("page must be integer") + } + query.PageNumber = i + continue + } + if strs[0] == "page_size" { + i, err := strconv.ParseInt(strs[1], 10, 64) + if err != nil { + return nil, ierror.New(nil). + WithCode(ierror.BadRequestCode). + WithMessage("page_size must be integer") + } + query.PageSize = i + continue + } + value, err := parsePattern(strs[1]) + if err != nil { + return nil, ierror.New(err). + WithCode(ierror.BadRequestCode). + WithMessage("invalid query string value: %s", strs[1]) + } + query.Keywords[strs[0]] = value + } + return query, nil +} + +func parsePattern(value string) (interface{}, error) { + // empty string + if len(value) == 0 { + return value, nil + } + switch value[0] { + case '~': + return parseFuzzyMatchValue(value) + case '[': + return parseRange(value) + case '{': + return parseOrList(value) + case '(': + return parseAndList(value) + default: + // others: exact match + return escapeValue(value), nil + } +} + +func parseFuzzyMatchValue(value string) (*FuzzyMatchValue, error) { + if len(value) < 2 || value[0] != '~' { + return nil, fmt.Errorf(`fuzzy match value must start with "~" and contain at least 1 other characters`) + } + return &FuzzyMatchValue{ + Value: value[1:], + }, nil +} + +func parseRange(value string) (*Range, error) { + length := len(value) + if value[length-1] != ']' || strings.Count(value, "~") != 1 { + return nil, fmt.Errorf(`range must start with "[", end with "]" and contains only one "~"`) + } + strs := strings.SplitN(value[1:length-1], "~", 2) + min := strings.TrimSpace(strs[0]) + max := strings.TrimSpace(strs[1]) + if len(min) == 0 && len(max) == 0 { + return nil, fmt.Errorf(`min and max at least one should be set in range'`) + } + r := &Range{} + if len(min) > 0 { + r.Min = parseValue(min) + } + if len(max) > 0 { + r.Max = parseValue(max) + } + return r, nil +} + +func parseOrList(value string) (*OrList, error) { + values, err := parseList(value, '{') + if err != nil { + return nil, err + } + ol := &OrList{} + for _, v := range values { + ol.Values = append(ol.Values, v) + } + return ol, nil +} + +func parseAndList(value string) (*AndList, error) { + values, err := parseList(value, '(') + if err != nil { + return nil, err + } + al := &AndList{} + for _, v := range values { + al.Values = append(al.Values, v) + } + return al, nil +} + +func parseList(value string, c rune) ([]interface{}, error) { + length := len(value) + if c == '{' && value[length-1] != '}' { + return nil, fmt.Errorf(`or list must start with "{" and end with "}"`) + } + if c == '(' && value[length-1] != ')' { + return nil, fmt.Errorf(`and list must start with "(" and end with ")"`) + } + var vs []interface{} + strs := strings.Split(value[1:length-1], " ") + for _, str := range strs { + v := parseValue(str) + if s, ok := v.(string); ok && len(s) == 0 { + continue + } + vs = append(vs, v) + } + return vs, nil +} + +// try to parse value as time first, then integer, and last string +func parseValue(value string) interface{} { + value = strings.TrimSpace(value) + // try to parse time + time, err := time.Parse("2006-01-02T15:04:05", value) + if err == nil { + return time + } + // try to parse integer + i, err := strconv.ParseInt(value, 10, 64) + if err == nil { + return i + } + // if the value isn't time and integer, treat it as string + return strings.Trim(value, `"'`) +} + +// escape the special character +func escapeValue(value string) string { + if len(value) > 0 && value[0] == '\\' { + return value[1:] + } + return value +} diff --git a/src/pkg/q/builder_test.go b/src/pkg/q/builder_test.go new file mode 100644 index 000000000..81966c1d9 --- /dev/null +++ b/src/pkg/q/builder_test.go @@ -0,0 +1,273 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package q + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestParseFuzzyMatchValue(t *testing.T) { + // empty string + value := "" + v, err := parseFuzzyMatchValue(value) + require.NotNil(t, err) + + // contain no other characters except "~" + value = "~" + v, err = parseFuzzyMatchValue(value) + require.NotNil(t, err) + + // valid fuzzy match value + value = "~a" + v, err = parseFuzzyMatchValue(value) + require.Nil(t, err) + assert.Equal(t, "a", v.Value) +} + +func TestParseRange(t *testing.T) { + // contains only "[" + value := "[" + v, err := parseRange(value) + require.NotNil(t, err) + + // contains no "," + value = "[]" + v, err = parseRange(value) + require.NotNil(t, err) + + // contains no other character + value = "[~]" + v, err = parseRange(value) + require.NotNil(t, err) + + // contains multiple "~" + value = "[~~]" + v, err = parseRange(value) + require.NotNil(t, err) + + // contains multiple char + value = "[1~2~3]" + v, err = parseRange(value) + require.NotNil(t, err) + + // valid value + value = "[1~]" + v, err = parseRange(value) + require.Nil(t, err) + assert.Equal(t, int64(1), v.Min.(int64)) + assert.Nil(t, v.Max) + + // valid value + value = "[~2]" + v, err = parseRange(value) + assert.Equal(t, int64(2), v.Max.(int64)) + assert.Nil(t, v.Min) + + // valid value + value = "[1~2]" + v, err = parseRange(value) + require.Nil(t, err) + assert.Equal(t, int64(1), v.Min.(int64)) + assert.Equal(t, int64(2), v.Max.(int64)) +} + +func TestParseOrList(t *testing.T) { + // invalid + value := "}{" + v, err := parseOrList(value) + require.NotNil(t, err) + + // valid value, contains no element + value = "{}" + v, err = parseOrList(value) + require.Nil(t, err) + assert.Len(t, v.Values, 0) + + // valid value, contains only one element + value = "{1}" + v, err = parseOrList(value) + require.Nil(t, err) + require.Len(t, v.Values, 1) + assert.Equal(t, int64(1), v.Values[0].(int64)) + + // valid value, contains multiple elements + value = "{1 2 3}" + v, err = parseOrList(value) + require.Nil(t, err) + require.Len(t, v.Values, 3) + assert.Equal(t, int64(1), v.Values[0].(int64)) + assert.Equal(t, int64(2), v.Values[1].(int64)) + assert.Equal(t, int64(3), v.Values[2].(int64)) +} + +func TestParseAndList(t *testing.T) { + // invalid + value := ")(" + v, err := parseAndList(value) + require.NotNil(t, err) + + // valid value, contains no element + value = "()" + v, err = parseAndList(value) + require.Nil(t, err) + assert.Len(t, v.Values, 0) + + // valid value, contains only one element + value = "(1)" + v, err = parseAndList(value) + require.Nil(t, err) + require.Len(t, v.Values, 1) + assert.Equal(t, int64(1), v.Values[0].(int64)) + + // valid value, contains multiple elements + value = "(1 2 3)" + v, err = parseAndList(value) + require.Nil(t, err) + require.Len(t, v.Values, 3) + assert.Equal(t, int64(1), v.Values[0].(int64)) + assert.Equal(t, int64(2), v.Values[1].(int64)) + assert.Equal(t, int64(3), v.Values[2].(int64)) +} + +func TestParseValue(t *testing.T) { + // time + value := "2020-03-04T17:08:23" + v := parseValue(value) + _, ok := v.(time.Time) + require.True(t, ok) + + // integer + value = "1" + v = parseValue(value) + i, ok := v.(int64) + require.True(t, ok) + assert.Equal(t, int64(1), i) + + // empty string + value = "" + v = parseValue(value) + str, ok := v.(string) + require.True(t, ok) + assert.Equal(t, "", str) + + // not empty string + value = "abc" + v = parseValue(value) + str, ok = v.(string) + require.True(t, ok) + assert.Equal(t, "abc", str) + + // not empty string + value = `"abc"` + v = parseValue(value) + str, ok = v.(string) + require.True(t, ok) + assert.Equal(t, "abc", str) +} + +func TestEscapeValue(t *testing.T) { + // empty string + value := "" + v := escapeValue(value) + assert.Equal(t, "", v) + + // string contains no special character + value = "abc" + v = escapeValue(value) + assert.Equal(t, "abc", v) + + // string starts with special character + value = `\~abc` + v = escapeValue(value) + assert.Equal(t, "~abc", v) +} + +func TestParsePattern(t *testing.T) { + // empty string + value := "" + v, err := parsePattern(value) + require.Nil(t, err) + _, ok := v.(string) + assert.True(t, ok) + + // fuzzy match + value = "~a" + v, err = parsePattern(value) + require.Nil(t, err) + _, ok = v.(*FuzzyMatchValue) + assert.True(t, ok) + + // range + value = "[1~3]" + v, err = parsePattern(value) + require.Nil(t, err) + _, ok = v.(*Range) + assert.True(t, ok) + + // or list + value = "{1 2}" + v, err = parsePattern(value) + require.Nil(t, err) + _, ok = v.(*OrList) + assert.True(t, ok) + + // and list + value = "(1 3)" + v, err = parsePattern(value) + require.Nil(t, err) + _, ok = v.(*AndList) + assert.True(t, ok) + + // exact match + value = "a" + v, err = parsePattern(value) + require.Nil(t, err) + _, ok = v.(string) + assert.True(t, ok) +} + +func TestBuild(t *testing.T) { + // empty string + q := `` + query, err := Build(q) + require.Nil(t, err) + assert.Nil(t, query) + + // contains only ";" + q = `,` + query, err = Build(q) + require.NotNil(t, err) + + // invalid page + q = `page=a` + query, err = Build(q) + require.NotNil(t, err) + + // invalid page + q = `page_size=a` + query, err = Build(q) + require.NotNil(t, err) + + // valid query string + q = `k=v,page=1,page_size=10` + query, err = Build(q) + require.Nil(t, err) + assert.Equal(t, int64(1), query.PageNumber) + assert.Equal(t, int64(10), query.PageSize) + assert.Equal(t, "v", query.Keywords["k"].(string)) +} diff --git a/src/pkg/q/query.go b/src/pkg/q/query.go index 904c61bbb..cf1b57700 100644 --- a/src/pkg/q/query.go +++ b/src/pkg/q/query.go @@ -47,3 +47,24 @@ func Copy(query *Query) *Query { } return q } + +// Range query +type Range struct { + Min interface{} + Max interface{} +} + +// AndList query +type AndList struct { + Values []interface{} +} + +// OrList query +type OrList struct { + Values []interface{} +} + +// FuzzyMatchValue query +type FuzzyMatchValue struct { + Value interface{} +} diff --git a/src/server/v2.0/handler/repository.go b/src/server/v2.0/handler/repository.go index b178c3e54..a0164f78d 100644 --- a/src/server/v2.0/handler/repository.go +++ b/src/server/v2.0/handler/repository.go @@ -53,22 +53,24 @@ func (r *repositoryAPI) ListRepositories(ctx context.Context, params operation.L if err != nil { return r.SendError(ctx, err) } + // set query - query := &q.Query{ - Keywords: map[string]interface{}{ - "ProjectID": project.ProjectID, - }, + var query *q.Query + if params.Q != nil { + query, err = q.Build(*params.Q) + if err != nil { + return r.SendError(ctx, err) + } } - // TODO support fuzzy match - if params.Name != nil { - query.Keywords["Name"] = *(params.Name) + + if query == nil { + query = &q.Query{Keywords: map[string]interface{}{}} } - if params.Page != nil { - query.PageNumber = *(params.Page) - } - if params.PageSize != nil { - query.PageSize = *(params.PageSize) + if query.Keywords == nil { + query.Keywords = map[string]interface{}{} } + query.Keywords["ProjectID"] = project.ProjectID + total, err := r.repoCtl.Count(ctx, query) if err != nil { return r.SendError(ctx, err)