Implement query string builder

This commit defines the API query string format and provides the builders to build query string to query model

Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin 2020-03-04 22:10:21 +08:00
parent 69119b6410
commit 8abb630b4c
10 changed files with 686 additions and 19 deletions

View File

@ -28,6 +28,8 @@ paths:
- $ref: '#/parameters/projectName'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- $ref: '#/parameters/query'
# TODO remove it
- name: name
in: query
description: Query the repositories by name
@ -500,6 +502,12 @@ paths:
'500':
$ref: '#/responses/500'
parameters:
query:
name: q
description: Query string to query resources. Supported query patterns are "exact match(k=v)", "fuzzy match(k=~v)", "range(k=[min~max])", "list with union releationship(k={v1 v2 v3})" and "list with intersetion relationship(k=(v1 v2 v3))". All of these query patterns should be put in the query string "q=xxx" and splitted by ",". e.g. q=k1=v1,k2=~v2,k3=[min~max]
in: query
type: string
required: false
requestId:
name: X-Request-Id
description: An unique ID for the request

View File

@ -16,14 +16,16 @@ package orm
import (
"context"
"reflect"
"strings"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/pkg/q"
)
// QuerySetter generates the query setter according to the query
func QuerySetter(ctx context.Context, model interface{}, query *q.Query) (orm.QuerySeter, error) {
// QuerySetter generates the query setter according to the query. "ignoredCols" is used to set the
// columns that will not be queried
func QuerySetter(ctx context.Context, model interface{}, query *q.Query, ignoredCols ...string) (orm.QuerySeter, error) {
ormer, err := FromContext(ctx)
if err != nil {
return nil, err
@ -32,7 +34,49 @@ func QuerySetter(ctx context.Context, model interface{}, query *q.Query) (orm.Qu
if query == nil {
return qs, nil
}
// the program will panic when querying the columns that doesn't exist
// list the supported columns first to avoid the panic
cols := listQueriableCols(model, ignoredCols...)
for k, v := range query.Keywords {
if _, exist := cols[k]; !exist {
continue
}
// fuzzy match
f, ok := v.(*q.FuzzyMatchValue)
if ok {
qs = qs.Filter(k+"__icontains", f.Value)
continue
}
// range
r, ok := v.(*q.Range)
if ok {
if r.Min != nil {
qs = qs.Filter(k+"__gte", r.Min)
}
if r.Max != nil {
qs = qs.Filter(k+"__lte", r.Max)
}
continue
}
// or list
ol, ok := v.(*q.OrList)
if ok {
qs = qs.Filter(k+"__in", ol.Values...)
continue
}
// and list
_, ok = v.(*q.OrList)
if ok {
// do nothing as and list needs to be handled by the logic of DAO
continue
}
// exact match
qs = qs.Filter(k, v)
}
if query.PageSize > 0 {
@ -44,6 +88,60 @@ func QuerySetter(ctx context.Context, model interface{}, query *q.Query) (orm.Qu
return qs, nil
}
// list the columns that can be queried
// e.g. for the following model the columns that can be queried are:
// "Field2", "customized_field2", "Field3" and "field3"
// type model struct{
// Field1 string `orm:"-"`
// Field2 string `orm:"column(customized_field2)"`
// Field3 string
// }
//
// set "ignoredCols" to ignore the specified columns
func listQueriableCols(model interface{}, ignoredCols ...string) map[string]struct{} {
if model == nil {
return nil
}
ignored := map[string]struct{}{}
for _, ig := range ignoredCols {
ignored[ig] = struct{}{}
}
cols := map[string]struct{}{}
t := reflect.Indirect(reflect.ValueOf(model)).Type()
for i := 0; i < t.NumField(); i++ {
field := t.Field(i)
orm := field.Tag.Get("orm")
if orm == "-" {
continue
}
colName := ""
for _, str := range strings.Split(orm, ";") {
if strings.HasPrefix(str, "column") {
str = strings.TrimPrefix(str, "column(")
str = strings.TrimSuffix(str, ")")
if len(str) > 0 {
colName = str
break
}
}
}
if len(colName) == 0 {
// TODO convert the field.Name to snake_case
}
if _, exist := ignored[colName]; exist {
continue
}
if _, exist := ignored[field.Name]; exist {
continue
}
if len(colName) != 0 {
cols[colName] = struct{}{}
}
cols[field.Name] = struct{}{}
}
return cols
}
// ParamPlaceholderForIn returns a string that contains placeholders for sql keyword "in"
// e.g. n=3, returns "?,?,?"
func ParamPlaceholderForIn(n int) string {

View File

@ -0,0 +1,61 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package orm
import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"testing"
)
func TestListQueriableCols(t *testing.T) {
type model struct {
Field1 string `orm:"column(field1)" json:"field1"`
Field2 string `orm:"column(customized_field2)"`
Field3 string
Field4 string `orm:"column(field4)"`
}
// without ignoring columns
cols := listQueriableCols(&model{})
require.Len(t, cols, 7)
_, exist := cols["Field1"]
assert.True(t, exist)
_, exist = cols["field1"]
assert.True(t, exist)
_, exist = cols["Field2"]
assert.True(t, exist)
_, exist = cols["customized_field2"]
assert.True(t, exist)
_, exist = cols["Field3"]
assert.True(t, exist)
_, exist = cols["Field4"]
assert.True(t, exist)
_, exist = cols["field4"]
assert.True(t, exist)
// with ignoring columns
cols = listQueriableCols(&model{}, "Field4")
require.Len(t, cols, 5)
_, exist = cols["Field1"]
assert.True(t, exist)
_, exist = cols["field1"]
assert.True(t, exist)
_, exist = cols["Field2"]
assert.True(t, exist)
_, exist = cols["customized_field2"]
assert.True(t, exist)
_, exist = cols["Field3"]
assert.True(t, exist)
}

View File

@ -253,8 +253,11 @@ func (d *dao) DeleteProjectBlob(ctx context.Context, projectID int64, blobIDs ..
if len(blobIDs) == 0 {
return nil
}
kw := q.KeyWords{"blob_id__in": blobIDs}
ol := &q.OrList{}
for _, blobID := range blobIDs {
ol.Values = append(ol.Values, blobID)
}
kw := q.KeyWords{"blob_id": ol}
qs, err := orm.QuerySetter(ctx, &models.ProjectBlob{}, q.New(kw))
if err != nil {
return err

View File

@ -160,7 +160,7 @@ func (suite *DaoTestSuite) TestListBlobs() {
suite.Len(blobs, 1)
}
blobs, err = suite.dao.ListBlobs(ctx, q.New(q.KeyWords{"digest__in": []string{digest1, digest2}}))
blobs, err = suite.dao.ListBlobs(ctx, q.New(q.KeyWords{"digest": &q.OrList{Values: []interface{}{digest1, digest2}}}))
if suite.Nil(err) {
suite.Len(blobs, 2)
}
@ -193,7 +193,11 @@ func (suite *DaoTestSuite) TestFindBlobsShouldUnassociatedWithProject() {
}
}
blobs, err := suite.dao.ListBlobs(ctx, q.New(q.KeyWords{"digest__in": blobDigests}))
ol := &q.OrList{}
for _, blobDigest := range blobDigests {
ol.Values = append(ol.Values, blobDigest)
}
blobs, err := suite.dao.ListBlobs(ctx, q.New(q.KeyWords{"digest": ol}))
suite.Nil(err)
suite.Len(blobs, 5)

View File

@ -126,7 +126,11 @@ func (m *manager) List(ctx context.Context, params ListParams) ([]*Blob, error)
}
if len(params.BlobDigests) > 0 {
kw["digest__in"] = params.BlobDigests
ol := &q.OrList{}
for _, blobDigest := range params.BlobDigests {
ol.Values = append(ol.Values, blobDigest)
}
kw["digest"] = ol
}
blobs, err := m.dao.ListBlobs(ctx, q.New(kw))

193
src/pkg/q/builder.go Normal file
View File

@ -0,0 +1,193 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package q
import (
"fmt"
ierror "github.com/goharbor/harbor/src/internal/error"
"strconv"
"strings"
"time"
)
// Build query sting into the Query model
// query string format: q=k=v,k=~v,k=[min~max],k={v1 v2 v3},k=(v1 v2 v3),page=1,page_size=10
// exact match: k=v
// fuzzy match: k=~v
// range: k=[min~max]
// or list: k={v1 v2 v3}
// and list: k=(v1 v2 v3)
func Build(q string) (*Query, error) {
if len(q) == 0 {
return nil, nil
}
query := &Query{Keywords: map[string]interface{}{}}
params := strings.Split(q, ",")
for _, param := range params {
strs := strings.SplitN(param, "=", 2)
if len(strs) != 2 || len(strs[0]) == 0 || len(strs[1]) == 0 {
return nil, ierror.New(nil).
WithCode(ierror.BadRequestCode).
WithMessage(`the query string must contain "=" and the key/value cannot be empty`)
}
if strs[0] == "page" {
i, err := strconv.ParseInt(strs[1], 10, 64)
if err != nil {
return nil, ierror.New(nil).
WithCode(ierror.BadRequestCode).
WithMessage("page must be integer")
}
query.PageNumber = i
continue
}
if strs[0] == "page_size" {
i, err := strconv.ParseInt(strs[1], 10, 64)
if err != nil {
return nil, ierror.New(nil).
WithCode(ierror.BadRequestCode).
WithMessage("page_size must be integer")
}
query.PageSize = i
continue
}
value, err := parsePattern(strs[1])
if err != nil {
return nil, ierror.New(err).
WithCode(ierror.BadRequestCode).
WithMessage("invalid query string value: %s", strs[1])
}
query.Keywords[strs[0]] = value
}
return query, nil
}
func parsePattern(value string) (interface{}, error) {
// empty string
if len(value) == 0 {
return value, nil
}
switch value[0] {
case '~':
return parseFuzzyMatchValue(value)
case '[':
return parseRange(value)
case '{':
return parseOrList(value)
case '(':
return parseAndList(value)
default:
// others: exact match
return escapeValue(value), nil
}
}
func parseFuzzyMatchValue(value string) (*FuzzyMatchValue, error) {
if len(value) < 2 || value[0] != '~' {
return nil, fmt.Errorf(`fuzzy match value must start with "~" and contain at least 1 other characters`)
}
return &FuzzyMatchValue{
Value: value[1:],
}, nil
}
func parseRange(value string) (*Range, error) {
length := len(value)
if value[length-1] != ']' || strings.Count(value, "~") != 1 {
return nil, fmt.Errorf(`range must start with "[", end with "]" and contains only one "~"`)
}
strs := strings.SplitN(value[1:length-1], "~", 2)
min := strings.TrimSpace(strs[0])
max := strings.TrimSpace(strs[1])
if len(min) == 0 && len(max) == 0 {
return nil, fmt.Errorf(`min and max at least one should be set in range'`)
}
r := &Range{}
if len(min) > 0 {
r.Min = parseValue(min)
}
if len(max) > 0 {
r.Max = parseValue(max)
}
return r, nil
}
func parseOrList(value string) (*OrList, error) {
values, err := parseList(value, '{')
if err != nil {
return nil, err
}
ol := &OrList{}
for _, v := range values {
ol.Values = append(ol.Values, v)
}
return ol, nil
}
func parseAndList(value string) (*AndList, error) {
values, err := parseList(value, '(')
if err != nil {
return nil, err
}
al := &AndList{}
for _, v := range values {
al.Values = append(al.Values, v)
}
return al, nil
}
func parseList(value string, c rune) ([]interface{}, error) {
length := len(value)
if c == '{' && value[length-1] != '}' {
return nil, fmt.Errorf(`or list must start with "{" and end with "}"`)
}
if c == '(' && value[length-1] != ')' {
return nil, fmt.Errorf(`and list must start with "(" and end with ")"`)
}
var vs []interface{}
strs := strings.Split(value[1:length-1], " ")
for _, str := range strs {
v := parseValue(str)
if s, ok := v.(string); ok && len(s) == 0 {
continue
}
vs = append(vs, v)
}
return vs, nil
}
// try to parse value as time first, then integer, and last string
func parseValue(value string) interface{} {
value = strings.TrimSpace(value)
// try to parse time
time, err := time.Parse("2006-01-02T15:04:05", value)
if err == nil {
return time
}
// try to parse integer
i, err := strconv.ParseInt(value, 10, 64)
if err == nil {
return i
}
// if the value isn't time and integer, treat it as string
return strings.Trim(value, `"'`)
}
// escape the special character
func escapeValue(value string) string {
if len(value) > 0 && value[0] == '\\' {
return value[1:]
}
return value
}

273
src/pkg/q/builder_test.go Normal file
View File

@ -0,0 +1,273 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package q
import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"testing"
"time"
)
func TestParseFuzzyMatchValue(t *testing.T) {
// empty string
value := ""
v, err := parseFuzzyMatchValue(value)
require.NotNil(t, err)
// contain no other characters except "~"
value = "~"
v, err = parseFuzzyMatchValue(value)
require.NotNil(t, err)
// valid fuzzy match value
value = "~a"
v, err = parseFuzzyMatchValue(value)
require.Nil(t, err)
assert.Equal(t, "a", v.Value)
}
func TestParseRange(t *testing.T) {
// contains only "["
value := "["
v, err := parseRange(value)
require.NotNil(t, err)
// contains no ","
value = "[]"
v, err = parseRange(value)
require.NotNil(t, err)
// contains no other character
value = "[~]"
v, err = parseRange(value)
require.NotNil(t, err)
// contains multiple "~"
value = "[~~]"
v, err = parseRange(value)
require.NotNil(t, err)
// contains multiple char
value = "[1~2~3]"
v, err = parseRange(value)
require.NotNil(t, err)
// valid value
value = "[1~]"
v, err = parseRange(value)
require.Nil(t, err)
assert.Equal(t, int64(1), v.Min.(int64))
assert.Nil(t, v.Max)
// valid value
value = "[~2]"
v, err = parseRange(value)
assert.Equal(t, int64(2), v.Max.(int64))
assert.Nil(t, v.Min)
// valid value
value = "[1~2]"
v, err = parseRange(value)
require.Nil(t, err)
assert.Equal(t, int64(1), v.Min.(int64))
assert.Equal(t, int64(2), v.Max.(int64))
}
func TestParseOrList(t *testing.T) {
// invalid
value := "}{"
v, err := parseOrList(value)
require.NotNil(t, err)
// valid value, contains no element
value = "{}"
v, err = parseOrList(value)
require.Nil(t, err)
assert.Len(t, v.Values, 0)
// valid value, contains only one element
value = "{1}"
v, err = parseOrList(value)
require.Nil(t, err)
require.Len(t, v.Values, 1)
assert.Equal(t, int64(1), v.Values[0].(int64))
// valid value, contains multiple elements
value = "{1 2 3}"
v, err = parseOrList(value)
require.Nil(t, err)
require.Len(t, v.Values, 3)
assert.Equal(t, int64(1), v.Values[0].(int64))
assert.Equal(t, int64(2), v.Values[1].(int64))
assert.Equal(t, int64(3), v.Values[2].(int64))
}
func TestParseAndList(t *testing.T) {
// invalid
value := ")("
v, err := parseAndList(value)
require.NotNil(t, err)
// valid value, contains no element
value = "()"
v, err = parseAndList(value)
require.Nil(t, err)
assert.Len(t, v.Values, 0)
// valid value, contains only one element
value = "(1)"
v, err = parseAndList(value)
require.Nil(t, err)
require.Len(t, v.Values, 1)
assert.Equal(t, int64(1), v.Values[0].(int64))
// valid value, contains multiple elements
value = "(1 2 3)"
v, err = parseAndList(value)
require.Nil(t, err)
require.Len(t, v.Values, 3)
assert.Equal(t, int64(1), v.Values[0].(int64))
assert.Equal(t, int64(2), v.Values[1].(int64))
assert.Equal(t, int64(3), v.Values[2].(int64))
}
func TestParseValue(t *testing.T) {
// time
value := "2020-03-04T17:08:23"
v := parseValue(value)
_, ok := v.(time.Time)
require.True(t, ok)
// integer
value = "1"
v = parseValue(value)
i, ok := v.(int64)
require.True(t, ok)
assert.Equal(t, int64(1), i)
// empty string
value = ""
v = parseValue(value)
str, ok := v.(string)
require.True(t, ok)
assert.Equal(t, "", str)
// not empty string
value = "abc"
v = parseValue(value)
str, ok = v.(string)
require.True(t, ok)
assert.Equal(t, "abc", str)
// not empty string
value = `"abc"`
v = parseValue(value)
str, ok = v.(string)
require.True(t, ok)
assert.Equal(t, "abc", str)
}
func TestEscapeValue(t *testing.T) {
// empty string
value := ""
v := escapeValue(value)
assert.Equal(t, "", v)
// string contains no special character
value = "abc"
v = escapeValue(value)
assert.Equal(t, "abc", v)
// string starts with special character
value = `\~abc`
v = escapeValue(value)
assert.Equal(t, "~abc", v)
}
func TestParsePattern(t *testing.T) {
// empty string
value := ""
v, err := parsePattern(value)
require.Nil(t, err)
_, ok := v.(string)
assert.True(t, ok)
// fuzzy match
value = "~a"
v, err = parsePattern(value)
require.Nil(t, err)
_, ok = v.(*FuzzyMatchValue)
assert.True(t, ok)
// range
value = "[1~3]"
v, err = parsePattern(value)
require.Nil(t, err)
_, ok = v.(*Range)
assert.True(t, ok)
// or list
value = "{1 2}"
v, err = parsePattern(value)
require.Nil(t, err)
_, ok = v.(*OrList)
assert.True(t, ok)
// and list
value = "(1 3)"
v, err = parsePattern(value)
require.Nil(t, err)
_, ok = v.(*AndList)
assert.True(t, ok)
// exact match
value = "a"
v, err = parsePattern(value)
require.Nil(t, err)
_, ok = v.(string)
assert.True(t, ok)
}
func TestBuild(t *testing.T) {
// empty string
q := ``
query, err := Build(q)
require.Nil(t, err)
assert.Nil(t, query)
// contains only ";"
q = `,`
query, err = Build(q)
require.NotNil(t, err)
// invalid page
q = `page=a`
query, err = Build(q)
require.NotNil(t, err)
// invalid page
q = `page_size=a`
query, err = Build(q)
require.NotNil(t, err)
// valid query string
q = `k=v,page=1,page_size=10`
query, err = Build(q)
require.Nil(t, err)
assert.Equal(t, int64(1), query.PageNumber)
assert.Equal(t, int64(10), query.PageSize)
assert.Equal(t, "v", query.Keywords["k"].(string))
}

View File

@ -47,3 +47,24 @@ func Copy(query *Query) *Query {
}
return q
}
// Range query
type Range struct {
Min interface{}
Max interface{}
}
// AndList query
type AndList struct {
Values []interface{}
}
// OrList query
type OrList struct {
Values []interface{}
}
// FuzzyMatchValue query
type FuzzyMatchValue struct {
Value interface{}
}

View File

@ -53,22 +53,24 @@ func (r *repositoryAPI) ListRepositories(ctx context.Context, params operation.L
if err != nil {
return r.SendError(ctx, err)
}
// set query
query := &q.Query{
Keywords: map[string]interface{}{
"ProjectID": project.ProjectID,
},
var query *q.Query
if params.Q != nil {
query, err = q.Build(*params.Q)
if err != nil {
return r.SendError(ctx, err)
}
// TODO support fuzzy match
if params.Name != nil {
query.Keywords["Name"] = *(params.Name)
}
if params.Page != nil {
query.PageNumber = *(params.Page)
if query == nil {
query = &q.Query{Keywords: map[string]interface{}{}}
}
if params.PageSize != nil {
query.PageSize = *(params.PageSize)
if query.Keywords == nil {
query.Keywords = map[string]interface{}{}
}
query.Keywords["ProjectID"] = project.ProjectID
total, err := r.repoCtl.Count(ctx, query)
if err != nil {
return r.SendError(ctx, err)