Upstream
/
harbor
mirror of https://github.com/goharbor/harbor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Introduce "sort" in query to provide a general solution for sorting 

Introduce "sort" in query to provide a general solution for sorting

Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin 2021-03-03 16:31:02 +08:00
parent b181d4df16
commit 506d1ad465
53 changed files with 700 additions and 540 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
api/v2.0/swagger.yaml
View File

@ -30,6 +30,7 @@ paths:
- $ref: '#/parameters/requestId'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- $ref: '#/parameters/sort'
- name: name
in: query
description: The name of project.
@ -249,6 +250,7 @@ paths:
- $ref: '#/parameters/requestId'
- $ref: '#/parameters/projectName'
- $ref: '#/parameters/query'
- $ref: '#/parameters/sort'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
responses:
@ -365,6 +367,7 @@ paths:
- $ref: '#/parameters/projectName'
- $ref: '#/parameters/repositoryName'
- $ref: '#/parameters/query'
- $ref: '#/parameters/sort'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- $ref: '#/parameters/acceptVulnerabilities'
@ -639,6 +642,7 @@ paths:
- $ref: '#/parameters/repositoryName'
- $ref: '#/parameters/reference'
- $ref: '#/parameters/query'
- $ref: '#/parameters/sort'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- name: with_signature
@ -936,6 +940,7 @@ paths:
parameters:
- $ref: '#/parameters/requestId'
- $ref: '#/parameters/query'
- $ref: '#/parameters/sort'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
responses:
@ -969,6 +974,7 @@ paths:
- $ref: '#/parameters/projectName'
- $ref: '#/parameters/requestId'
- $ref: '#/parameters/query'
- $ref: '#/parameters/sort'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
responses:
@ -1056,6 +1062,7 @@ paths:
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- $ref: '#/parameters/query'
- $ref: '#/parameters/sort'
responses:
'200':
description: Success
@ -1223,6 +1230,7 @@ paths:
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- $ref: '#/parameters/query'
- $ref: '#/parameters/sort'
responses:
'200':
description: List preheat policies success
@ -1368,6 +1376,7 @@ paths:
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- $ref: '#/parameters/query'
- $ref: '#/parameters/sort'
responses:
'200':
description: List executions success
@ -1464,6 +1473,7 @@ paths:
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- $ref: '#/parameters/query'
- $ref: '#/parameters/sort'
responses:
'200':
description: List tasks success
@ -1560,6 +1570,7 @@ paths:
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- $ref: '#/parameters/query'
- $ref: '#/parameters/sort'
tags:
- robotv1
operationId: ListRobotV1
@ -1717,6 +1728,7 @@ paths:
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- $ref: '#/parameters/query'
- $ref: '#/parameters/sort'
responses:
'200':
description: Success
@ -1847,6 +1859,7 @@ paths:
parameters:
- $ref: '#/parameters/requestId'
- $ref: '#/parameters/query'
- $ref: '#/parameters/sort'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
responses:
@ -2126,6 +2139,7 @@ paths:
- replication
operationId: listReplicationExecutions
parameters:
- $ref: '#/parameters/sort'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- name: policy_id
@ -2246,6 +2260,7 @@ paths:
- replication
operationId: listReplicationTasks
parameters:
- $ref: '#/parameters/sort'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- name: id
@ -2460,6 +2475,7 @@ paths:
operationId: getGCHistory
parameters:
- $ref: '#/parameters/query'
- $ref: '#/parameters/sort'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
responses:
@ -3240,6 +3256,29 @@ parameters:
in: query
type: string
required: false
sort:
name: sort
description: Sort the resource list in ascending or descending order. e.g. sort by field1 in ascending orderr and field2 in descending order with "sort=field1,-field2"
in: query
type: string
required: false
page:
name: page
in: query
type: integer
format: int64
required: false
description: The page number
default: 1
pageSize:
name: page_size
in: query
type: integer
format: int64
required: false
description: The size of per page
default: 10
maximum: 100
requestId:
name: X-Request-Id
description: An unique ID for the request
@ -3305,29 +3344,6 @@ parameters:
description: The name of the tag
required: true
type: string
page:
name: page
in: query
type: integer
format: int64
required: false
description: The page number
default: 1
pageSize:
name: page_size
in: query
type: integer
format: int64
required: false
description: The size of per page
default: 10
maximum: 100
sort:
name: sort
in: query
type: string
required: false
description: The order by fields of the query, the format is '+field1,-field2'.
instanceName:
name: preheat_instance_name
in: path

2
src/common/models/project.go
View File

@ -38,7 +38,7 @@ const (
type Project struct {
ProjectID int64 `orm:"pk;auto;column(project_id)" json:"project_id"`
OwnerID int `orm:"column(owner_id)" json:"owner_id"`
Name string `orm:"column(name)" json:"name"`
Name string `orm:"column(name)" json:"name" sort:"default"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
Deleted bool `orm:"column(deleted)" json:"deleted"`

2
src/common/models/repo.go
View File

@ -34,7 +34,7 @@ type RepoRecord struct {
Description string `orm:"column(description)" json:"description"`
PullCount int64 `orm:"column(pull_count)" json:"pull_count"`
StarCount int64 `orm:"column(star_count)" json:"star_count"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time" sort:"default:desc"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}

2
src/common/models/user.go
View File

@ -24,7 +24,7 @@ const UserTable = "harbor_user"
// User holds the details of a user.
type User struct {
UserID int `orm:"pk;auto;column(user_id)" json:"user_id"`
Username string `orm:"column(username)" json:"username"`
Username string `orm:"column(username)" json:"username" sort:"default"`
Email string `orm:"column(email)" json:"email"`
Password string `orm:"column(password)" json:"password"`
PasswordVersion string `orm:"column(password_version)" json:"password_version"`

2
src/core/api/registry.go
View File

@ -178,7 +178,7 @@ func (t *RegistryAPI) List() {
queryStr = fmt.Sprintf("name=~%s", name)
}
}
query, err := q.Build(queryStr, 0, 0)
query, err := q.Build(queryStr, "", 0, 0)
if err != nil {
t.SendError(err)
return

2
src/core/api/search.go
View File

@ -51,8 +51,6 @@ func (s *SearchAPI) Get() {
keyword := s.GetString("q")
query := q.New(q.KeyWords{})
query.Sorting = "name"
if keyword != "" {
query.Keywords["name"] = &q.FuzzyMatchValue{Value: keyword}
}

4
src/jobservice/job/impl/gc/garbage_collection.go
View File

@ -442,7 +442,9 @@ func (gc *GarbageCollector) removeUntaggedBlobs(ctx job.Context) {
},
PageNumber: 1,
PageSize: int64(ps),
Sorting: "id",
Sorts: []*q.Sort{
q.NewSort("id", false),
},
}
blobs, err := gc.blobMgr.List(ctx.SystemContext(), q)
if err != nil {

217
src/lib/orm/metadata.go Normal file
View File

@ -0,0 +1,217 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package orm
import (
"context"
"reflect"
"strings"
"sync"
"unicode"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/lib/q"
)
var (
// cache the parsed models
cache = sync.Map{}
)
type key struct {
Name string
Filterable bool
FilterFunc func(context.Context, orm.QuerySeter, string, interface{}) orm.QuerySeter
Sortable bool
}
type metadata struct {
Keys map[string]*key
DefaultSort *q.Sort
}
func (m *metadata) Filterable(key string) (*key, bool) {
k, exist := m.Keys[key]
return k, exist
}
func (m *metadata) Sortable(key string) bool {
k, exist := m.Keys[key]
if !exist {
return false
}
return k.Sortable
}
// parse the definition of the provided model(fields/methods/annotations) and return the parsed metadata
func parseModel(model interface{}) *metadata {
// pointer type
ptr := reflect.TypeOf(model)
// struct type
t := ptr.Elem()
// get the metadata from cache first
fullName := getFullName(t)
cacheMetadata, exist := cache.Load(fullName)
if exist {
return cacheMetadata.(*metadata)
}
// pointer value
v := reflect.ValueOf(model)
metadata := &metadata{
Keys: map[string]*key{},
}
// parse fields of the provided model
for i := 0; i < t.NumField(); i++ {
field := t.Field(i)
orm := field.Tag.Get("orm")
// isn't the database column, skip
if orm == "-" {
continue
}
filterable := parseFilterable(field)
defaultSort, sortable := parseSortable(field)
column := parseColumn(field)
metadata.Keys[field.Name] = &key{
Name: field.Name,
Filterable: filterable,
Sortable: sortable,
}
metadata.Keys[column] = &key{
Name: column,
Filterable: filterable,
Sortable: sortable,
}
if defaultSort != nil {
metadata.DefaultSort = defaultSort
}
}
// parse methods of the provided model
for i := 0; i < ptr.NumMethod(); i++ {
methodName := ptr.Method(i).Name
if !strings.HasPrefix(methodName, "FilterBy") {
continue
}
methodValue := v.MethodByName(methodName)
if !methodValue.IsValid() {
continue
}
filterFunc, ok := methodValue.Interface().(func(context.Context, orm.QuerySeter, string, interface{}) orm.QuerySeter)
if !ok {
continue
}
field := strings.TrimPrefix(methodName, "FilterBy")
metadata.Keys[field] = &key{
Name: field,
Filterable: true,
FilterFunc: filterFunc,
}
snakeCaseField := snakeCase(field)
metadata.Keys[snakeCaseField] = &key{
Name: snakeCaseField,
Filterable: true,
FilterFunc: filterFunc,
}
}
cache.Store(fullName, metadata)
return metadata
}
// parseFilterable parses whether the field is filterable according to the field annotation
// For the following struct definition, "Field1" isn't filterable and "Field2" is filterable
// type Model struct {
// Field1 string `filter:"false"`
// Field2 string
// }
func parseFilterable(field reflect.StructField) bool {
return field.Tag.Get("filter") != "false"
}
// parseSortable parses whether the field is sortable according to the field annotation
// If the field is sortable and is also specified as the default sort, return a q.Sort model as well
// For the following struct definition, "Field1" isn't sortable and "Field2", "Field2", "Field4", "Field5" are all sortable
// type Model struct {
// Field1 string `sort:"false"`
// Field2 string `sort:"true;default"`
// Field3 string `sort:"true;default:desc"`
// Field4 string `sort:"default"`
// Field5 string
// }
func parseSortable(field reflect.StructField) (*q.Sort, bool) {
var defaultSort *q.Sort
for _, item := range strings.Split(field.Tag.Get("sort"), ";") {
// isn't sortable, return directly
if item == "false" {
return nil, false
}
if !strings.HasPrefix(item, "default") {
continue
}
defaultSort = &q.Sort{
Key: field.Name,
DESC: false,
}
if strings.TrimPrefix(item, "default") == ":desc" {
defaultSort.DESC = true
}
}
return defaultSort, true
}
// parseColumn parses the column name according to the field annotation
// type Model struct {
// Field1 string `orm:"column(customized_field1)"`
// Field2 string
// }
// It returns "customized_field1" for "Field1" and returns "field2" for "Field2"
func parseColumn(field reflect.StructField) string {
column := ""
for _, item := range strings.Split(field.Tag.Get("orm"), ";") {
if !strings.HasPrefix(item, "column") {
continue
}
item = strings.TrimPrefix(item, "column(")
item = strings.TrimSuffix(item, ")")
column = item
break
}
if len(column) == 0 {
column = snakeCase(field.Name)
}
return column
}
// convert string to snake case
func snakeCase(str string) string {
delim := '_'
runes := []rune(str)
var out []rune
for i := 0; i < len(runes); i++ {
if i > 0 &&
(unicode.IsUpper(runes[i])) &&
((i+1 < len(runes) && unicode.IsLower(runes[i+1])) || unicode.IsLower(runes[i-1])) {
out = append(out, delim)
}
out = append(out, unicode.ToLower(runes[i]))
}
return string(out)
}

123
src/lib/orm/metadata_test.go Normal file
View File

@ -0,0 +1,123 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package orm
import (
"context"
"testing"
"github.com/astaxie/beego/orm"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type foo struct {
Field1 string `orm:"-"`
Field2 string `orm:"column(customized_field2)" filter:"false"`
Field3 string `sort:"false"`
Field4 string `sort:"default:desc"`
}
func (f *foo) FilterByField5(context.Context, orm.QuerySeter, string, interface{}) orm.QuerySeter {
return nil
}
func (f *foo) OtherFunc() {}
func TestParseQueryObject(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
metadata := parseModel(&foo{})
require.NotNil(metadata)
require.Len(metadata.Keys, 8)
key, exist := metadata.Keys["Field2"]
require.True(exist)
assert.Equal("Field2", key.Name)
assert.False(key.Filterable)
assert.True(key.Sortable)
key, exist = metadata.Keys["customized_field2"]
require.True(exist)
assert.Equal("customized_field2", key.Name)
assert.False(key.Filterable)
assert.True(key.Sortable)
key, exist = metadata.Keys["Field3"]
require.True(exist)
assert.Equal("Field3", key.Name)
assert.True(key.Filterable)
assert.False(key.Sortable)
key, exist = metadata.Keys["field3"]
require.True(exist)
assert.Equal("field3", key.Name)
assert.True(key.Filterable)
assert.False(key.Sortable)
key, exist = metadata.Keys["Field4"]
require.True(exist)
assert.Equal("Field4", key.Name)
assert.True(key.Filterable)
assert.True(key.Sortable)
key, exist = metadata.Keys["field4"]
require.True(exist)
assert.Equal("field4", key.Name)
assert.True(key.Filterable)
assert.True(key.Sortable)
key, exist = metadata.Keys["Field5"]
require.True(exist)
assert.Equal("Field5", key.Name)
assert.True(key.Filterable)
assert.False(key.Sortable)
key, exist = metadata.Keys["field5"]
require.True(exist)
assert.Equal("field5", key.Name)
assert.True(key.Filterable)
assert.False(key.Sortable)
require.NotNil(metadata.DefaultSort)
assert.Equal("Field4", metadata.DefaultSort.Key)
assert.True(metadata.DefaultSort.DESC)
}
func Test_snakeCase(t *testing.T) {
type args struct {
str string
}
tests := []struct {
name string
args args
want string
}{
{"ProjectID", args{"ProjectID"}, "project_id"},
{"project_id", args{"project_id"}, "project_id"},
{"RepositoryName", args{"RepositoryName"}, "repository_name"},
{"repository_name", args{"repository_name"}, "repository_name"},
{"ProfileURL", args{"ProfileURL"}, "profile_url"},
{"City", args{"City"}, "city"},
{"Address1", args{"Address1"}, "address1"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := snakeCase(tt.args.str); got != tt.want {
t.Errorf("snakeCase() = %v, want %v", got, tt.want)
}
})
}
}

31
src/lib/orm/orm.go
View File

@ -18,6 +18,7 @@ import (
"context"
"errors"
"fmt"
"github.com/goharbor/harbor/src/common/dao"
"strconv"
"strings"
@ -25,6 +26,21 @@ import (
"github.com/goharbor/harbor/src/lib/log"
)
// NewCondition alias function of orm.NewCondition
var NewCondition = orm.NewCondition
// Condition alias to orm.Condition
type Condition = orm.Condition
// Params alias to orm.Params
type Params = orm.Params
// ParamsList alias to orm.ParamsList
type ParamsList = orm.ParamsList
// QuerySeter alias to orm.QuerySeter
type QuerySeter = orm.QuerySeter
// RegisterModel ...
func RegisterModel(models ...interface{}) {
orm.RegisterModel(models...)
@ -174,3 +190,18 @@ func CreateInClause(ctx context.Context, sql string, args ...interface{}) (strin
// when concat the in clause directly
return fmt.Sprintf(`IN (%s)`, strings.Join(idStrs, ",")), nil
}
// Escape special characters
func Escape(str string) string {
return dao.Escape(str)
}
// ParamPlaceholderForIn returns a string that contains placeholders for sql keyword "in"
// e.g. n=3, returns "?,?,?"
func ParamPlaceholderForIn(n int) string {
placeholders := []string{}
for i := 0; i < n; i++ {
placeholders = append(placeholders, "?")
}
return strings.Join(placeholders, ",")
}

335
src/lib/orm/query.go
View File

@ -16,68 +16,33 @@ package orm
import (
"context"
"fmt"
"reflect"
"strings"
"sync"
"unicode"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/q"
)
// NewCondition alias function of orm.NewCondition
var NewCondition = orm.NewCondition
// Condition alias to orm.Condition
type Condition = orm.Condition
// Params alias to orm.Params
type Params = orm.Params
// ParamsList alias to orm.ParamsList
type ParamsList = orm.ParamsList
// QuerySeter alias to orm.QuerySeter
type QuerySeter = orm.QuerySeter
// Escape special characters
func Escape(str string) string {
return dao.Escape(str)
}
// ParamPlaceholderForIn returns a string that contains placeholders for sql keyword "in"
// e.g. n=3, returns "?,?,?"
func ParamPlaceholderForIn(n int) string {
placeholders := []string{}
for i := 0; i < n; i++ {
placeholders = append(placeholders, "?")
}
return strings.Join(placeholders, ",")
}
// QuerySetter generates the query setter according to the query. "ignoredCols" is used to set the columns that will not be queried.
// Currently, it supports two ways to generate the query setter, the first one is to generate by the fields of the model,
// and the second one is to generate by the methods their name begins with `FilterBy` of the model.
// e.g. for the following model the queriable fields are :
// "Field2", "customized_field2", "Field3", "field3" and "Field4" (or "field4").
// QuerySetter generates the query setter according to the provided model and query.
// e.g.
// type Foo struct{
// Field1 string `orm:"-"`
// Field2 string `orm:"column(customized_field2)"`
// Field3 string
// Field1 string `orm:"-"` // can not filter/sort
// Field2 string `orm:"column(customized_field2)"` // support filter by "Field2", "customized_field2"
// Field3 string `sort:"false"` // cannot be sorted
// Field4 string `sort:"default:desc"` // the default field/order(asc/desc) to sort if no sorting specified in query.
// Field5 string `filter:"false"` // cannot be filtered
// }
//
// func (f *Foo) FilterByField4(ctx context.Context, qs orm.QuerySeter, key string, value interface{}) orm.QuerySeter {
// // The value is the raw value of key in q.Query
// // support filter by "Field6", "field6"
// func (f *Foo) FilterByField6(ctx context.Context, qs orm.QuerySetter, key string, value interface{}) orm.QuerySetter {
// ...
// return qs
// }
func QuerySetter(ctx context.Context, model interface{}, query *q.Query, ignoredCols ...string) (orm.QuerySeter, error) {
val := reflect.ValueOf(model)
if val.Kind() != reflect.Ptr {
return nil, errors.Errorf("<orm.QuerySetter> cannot use non-ptr model struct `%s`", getFullName(reflect.Indirect(val).Type()))
func QuerySetter(ctx context.Context, model interface{}, query *q.Query) (orm.QuerySeter, error) {
t := reflect.TypeOf(model)
if t.Kind() != reflect.Ptr {
return nil, fmt.Errorf("<orm.QuerySetter> cannot use non-ptr model struct `%s`", getFullName(t.Elem()))
}
ormer, err := FromContext(ctx)
if err != nil {
return nil, err
@ -87,190 +52,116 @@ func QuerySetter(ctx context.Context, model interface{}, query *q.Query, ignored
return qs, nil
}
ignored := map[string]bool{}
for _, col := range ignoredCols {
ignored[col] = true
}
metadata := parseModel(model)
// set filters
qs = setFilters(ctx, qs, query, metadata)
columns := queriableColumns(model)
methods := queriableMethods(model)
for k, v := range query.Keywords {
field := strings.SplitN(k, orm.ExprSep, 2)[0]
if ignored[field] {
continue
}
if columns[field] {
qs = queryByColumn(qs, k, v)
} else if method, ok := methods[snakeCase(field)]; ok {
qs = queryByMethod(ctx, qs, k, v, method, val)
}
}
// sorting
qs = setSorts(qs, query, metadata)
// pagination
if query.PageSize > 0 {
qs = qs.Limit(query.PageSize)
if query.PageNumber > 0 {
qs = qs.Offset(query.PageSize * (query.PageNumber - 1))
}
}
return qs, nil
}
// QuerySetterForCount creates the query setter used for count with the sort and pagination information ignored
func QuerySetterForCount(ctx context.Context, model interface{}, query *q.Query, ignoredCols ...string) (orm.QuerySeter, error) {
query = q.MustClone(query)
query.Sorts = nil
query.PageSize = 0
query.PageNumber = 0
return QuerySetter(ctx, model, query)
}
// set filters according to the query
func setFilters(ctx context.Context, qs orm.QuerySeter, query *q.Query, meta *metadata) orm.QuerySeter {
for key, value := range query.Keywords {
// The "strings.SplitN()" here is a workaround for the incorrect usage of query which should be avoided
// e.g. use the query with the knowledge of underlying ORM implementation, the "OrList" should be used instead:
// https://github.com/goharbor/harbor/blob/v2.2.0/src/controller/project/controller.go#L348
k := strings.SplitN(key, orm.ExprSep, 2)[0]
mk, filterable := meta.Filterable(k)
if !filterable {
// This is a workaround for the unsuitable usage of query, the keyword format for field and method should be consistent
// e.g. "ArtifactDigest" or the snake case format "artifact_digest" should be used instead:
// https://github.com/goharbor/harbor/blob/v2.2.0/src/controller/blob/controller.go#L233
mk, filterable = meta.Filterable(snakeCase(k))
if !filterable {
continue
}
}
// filter function defined, use it directly
if mk.FilterFunc != nil {
qs = mk.FilterFunc(ctx, qs, key, value)
continue
}
// fuzzy match
if f, ok := value.(*q.FuzzyMatchValue); ok {
qs = qs.Filter(key+"__icontains", Escape(f.Value))
continue
}
// range
if r, ok := value.(*q.Range); ok {
if r.Min != nil {
qs = qs.Filter(key+"__gte", r.Min)
}
if r.Max != nil {
qs = qs.Filter(key+"__lte", r.Max)
}
continue
}
// or list
if ol, ok := value.(*q.OrList); ok {
if len(ol.Values) > 0 {
qs = qs.Filter(key+"__in", ol.Values...)
}
continue
}
// and list
if _, ok := value.(*q.AndList); ok {
// do nothing as and list needs to be handled by the logic of DAO
continue
}
// exact match
qs = qs.Filter(key, value)
}
return qs
}
// set sorts according to the query
func setSorts(qs orm.QuerySeter, query *q.Query, meta *metadata) orm.QuerySeter {
var sortings []string
for _, sort := range query.Sorts {
if !meta.Sortable(sort.Key) {
continue
}
sorting := sort.Key
if sort.DESC {
sorting = fmt.Sprintf("-%s", sorting)
}
sortings = append(sortings, sorting)
}
// if no sorts are specified, apply the default sort setting if exists
if len(sortings) == 0 && meta.DefaultSort != nil {
sorting := meta.DefaultSort.Key
if meta.DefaultSort.DESC {
sorting = fmt.Sprintf("-%s", sorting)
}
sortings = append(sortings, sorting)
}
if len(sortings) > 0 {
qs = qs.OrderBy(sortings...)
}
return qs
}
// get reflect.Type name with package path.
func getFullName(typ reflect.Type) string {
return typ.PkgPath() + "." + typ.Name()
}
// convert string to snake case
func snakeCase(str string) string {
delim := '_'
runes := []rune(str)
var out []rune
for i := 0; i < len(runes); i++ {
if i > 0 &&
(unicode.IsUpper(runes[i])) &&
((i+1 < len(runes) && unicode.IsLower(runes[i+1])) || unicode.IsLower(runes[i-1])) {
out = append(out, delim)
}
out = append(out, unicode.ToLower(runes[i]))
}
return string(out)
}
func queryByColumn(qs orm.QuerySeter, key string, value interface{}) orm.QuerySeter {
// fuzzy match
if f, ok := value.(*q.FuzzyMatchValue); ok {
return qs.Filter(key+"__icontains", Escape(f.Value))
}
// range
if r, ok := value.(*q.Range); ok {
if r.Min != nil {
qs = qs.Filter(key+"__gte", r.Min)
}
if r.Max != nil {
qs = qs.Filter(key+"__lte", r.Max)
}
return qs
}
// or list
if ol, ok := value.(*q.OrList); ok {
if len(ol.Values) > 0 {
qs = qs.Filter(key+"__in", ol.Values...)
}
return qs
}
// and list
if _, ok := value.(*q.AndList); ok {
// do nothing as and list needs to be handled by the logic of DAO
return qs
}
// exact match
return qs.Filter(key, value)
}
func queryByMethod(ctx context.Context, qs orm.QuerySeter, key string, value interface{}, methodName string, reflectVal reflect.Value) orm.QuerySeter {
if mv := reflectVal.MethodByName(methodName); mv.IsValid() {
switch method := mv.Interface().(type) {
case func(context.Context, orm.QuerySeter, string, interface{}) orm.QuerySeter:
return method(ctx, qs, key, value)
default:
return qs
}
}
return qs
}
var (
cache = sync.Map{}
)
// get model fields which are columns in orm
// e.g. for the following model the columns that can be queried are:
// "Field2", "customized_field2", "Field3" and "field3"
// type model struct{
// Field1 string `orm:"-"`
// Field2 string `orm:"column(customized_field2)"`
// Field3 string
// }
func queriableColumns(model interface{}) map[string]bool {
typ := reflect.Indirect(reflect.ValueOf(model)).Type()
key := getFullName(typ) + "-columns"
value, ok := cache.Load(key)
if ok {
return value.(map[string]bool)
}
cols := map[string]bool{}
defer func() {
cache.Store(key, cols)
}()
for i := 0; i < typ.NumField(); i++ {
field := typ.Field(i)
orm := field.Tag.Get("orm")
if orm == "-" {
continue
}
colName := ""
for _, str := range strings.Split(orm, ";") {
if strings.HasPrefix(str, "column") {
str = strings.TrimPrefix(str, "column(")
str = strings.TrimSuffix(str, ")")
if len(str) > 0 {
colName = str
break
}
}
}
if colName == "" {
colName = snakeCase(field.Name)
}
cols[colName] = true
cols[field.Name] = true
}
return cols
}
// get model methods which begin with `FilterBy`
func queriableMethods(model interface{}) map[string]string {
val := reflect.ValueOf(model)
key := getFullName(reflect.Indirect(val).Type()) + "-methods"
value, ok := cache.Load(key)
if ok {
return value.(map[string]string)
}
methods := map[string]string{}
defer func() {
cache.Store(key, methods)
}()
prefix := "FilterBy"
typ := val.Type()
for i := 0; i < typ.NumMethod(); i++ {
name := typ.Method(i).Name
if !strings.HasPrefix(name, prefix) {
continue
}
field := snakeCase(strings.TrimPrefix(name, prefix))
if field != "" {
methods[field] = name
}
}
return methods
}

109
src/lib/orm/query_test.go
View File

@ -1,109 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package orm
import (
"reflect"
"testing"
)
func Test_snakeCase(t *testing.T) {
type args struct {
str string
}
tests := []struct {
name string
args args
want string
}{
{"ProjectID", args{"ProjectID"}, "project_id"},
{"project_id", args{"project_id"}, "project_id"},
{"RepositoryName", args{"RepositoryName"}, "repository_name"},
{"repository_name", args{"repository_name"}, "repository_name"},
{"ProfileURL", args{"ProfileURL"}, "profile_url"},
{"City", args{"City"}, "city"},
{"Address1", args{"Address1"}, "address1"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := snakeCase(tt.args.str); got != tt.want {
t.Errorf("snakeCase() = %v, want %v", got, tt.want)
}
})
}
}
type Bar struct {
Field1 string `orm:"-"`
Field2 string `orm:"column(customized_field2)"`
Field3 string
FirstName string
}
func (Bar) Foo() {}
func (bar *Bar) FilterBy() {}
func (bar *Bar) FilterByField4() {}
func Test_queriableColumns(t *testing.T) {
toWant := func(fields ...string) map[string]bool {
want := map[string]bool{}
for _, field := range fields {
want[field] = true
}
return want
}
type args struct {
model interface{}
}
tests := []struct {
name string
args args
want map[string]bool
}{
{"bar", args{&Bar{}}, toWant("Field2", "customized_field2", "Field3", "field3", "FirstName", "first_name")},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := queriableColumns(tt.args.model); !reflect.DeepEqual(got, tt.want) {
t.Errorf("queriableColumns() = %v, want %v", got, tt.want)
}
})
}
}
func Test_queriableMethods(t *testing.T) {
type args struct {
model interface{}
}
tests := []struct {
name string
args args
want map[string]string
}{
{"bar", args{&Bar{}}, map[string]string{"field4": "FilterByField4"}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := queriableMethods(tt.args.model); !reflect.DeepEqual(got, tt.want) {
t.Errorf("queriableMethods() = %v, want %v", got, tt.want)
}
})
}
}

49
src/lib/q/builder.go
View File

@ -16,29 +16,41 @@ package q
import (
"fmt"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"net/url"
"strconv"
"strings"
"time"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
)
// Build query sting and pagination information into the Query model
// Build query sting, sort and pagination information into the Query model
// query string format: q=k=v,k=~v,k=[min~max],k={v1 v2 v3},k=(v1 v2 v3)
// exact match: k=v
// fuzzy match: k=~v
// range: k=[min~max]
// or list: k={v1 v2 v3}
// and list: k=(v1 v2 v3)
func Build(q string, pageNumber, pageSize int64) (*Query, error) {
query := &Query{
// sort format: sort=k1,-k2
func Build(q, sort string, pageNumber, pageSize int64) (*Query, error) {
keywords, err := parseKeywords(q)
if err != nil {
return nil, err
}
sorts := parseSorting(sort)
return &Query{
Keywords: keywords,
Sorts: sorts,
PageNumber: pageNumber,
PageSize: pageSize,
Keywords: map[string]interface{}{},
}
}, nil
}
func parseKeywords(q string) (map[string]interface{}, error) {
keywords := map[string]interface{}{}
if len(q) == 0 {
return query, nil
return keywords, nil
}
// try to escaped the 'q=tags%3Dnil' when to filter tags.
if unescapedQuery, err := url.QueryUnescape(q); err == nil {
@ -60,9 +72,26 @@ func Build(q string, pageNumber, pageSize int64) (*Query, error) {
WithCode(errors.BadRequestCode).
WithMessage("invalid query string value: %s", strs[1])
}
query.Keywords[strs[0]] = value
keywords[strs[0]] = value
}
return query, nil
return keywords, nil
}
func parseSorting(sort string) []*Sort {
var sorts []*Sort
for _, sorting := range strings.Split(sort, ",") {
key := sorting
desc := false
if strings.HasPrefix(sorting, "-") {
key = strings.TrimPrefix(sorting, "-")
desc = true
}
sorts = append(sorts, &Sort{
Key: key,
DESC: desc,
})
}
return sorts
}
func parsePattern(value string) (interface{}, error) {

22
src/lib/q/builder_test.go
View File

@ -241,32 +241,26 @@ func TestParsePattern(t *testing.T) {
assert.True(t, ok)
}
func TestBuild(t *testing.T) {
func TestParseKeywords(t *testing.T) {
// empty string
q := ``
query, err := Build(q, 1, 10)
keywords, err := parseKeywords(q)
require.Nil(t, err)
require.NotNil(t, query)
assert.Equal(t, int64(1), query.PageNumber)
assert.Equal(t, int64(10), query.PageSize)
require.NotNil(t, keywords)
// contains only ","
q = `,`
query, err = Build(q, 1, 10)
keywords, err = parseKeywords(q)
require.NotNil(t, err)
// valid query string
q = `k=v`
query, err = Build(q, 1, 10)
keywords, err = parseKeywords(q)
require.Nil(t, err)
assert.Equal(t, int64(1), query.PageNumber)
assert.Equal(t, int64(10), query.PageSize)
assert.Equal(t, "v", query.Keywords["k"].(string))
assert.Equal(t, "v", keywords["k"].(string))
q = `q=tags%3Dnil`
query, err = Build(q, 1, 10)
keywords, err = parseKeywords(q)
require.Nil(t, err)
assert.Equal(t, int64(1), query.PageNumber)
assert.Equal(t, int64(10), query.PageSize)
assert.Equal(t, "tags=nil", query.Keywords["q"].(string))
assert.Equal(t, "tags=nil", keywords["q"].(string))
}

63
src/lib/q/query.go
View File

@ -19,22 +19,24 @@ type KeyWords = map[string]interface{}
// Query parameters
type Query struct {
// Filter list
Keywords KeyWords
// Sort list
Sorts []*Sort
// Page number
PageNumber int64
// Page size
PageSize int64
// List of key words
Keywords KeyWords
// Sorting
// Deprecate, use "Sorts" instead
Sorting string
}
// First make the query only fetch the first one record in the sorting order
func (q *Query) First(sorting ...string) *Query {
func (q *Query) First(sorting ...*Sort) *Query {
q.PageNumber = 1
q.PageSize = 1
if len(sorting) > 0 {
q.Sorting = sorting[0]
q.Sorts = append(q.Sorts, sorting...)
}
return q
@ -54,14 +56,26 @@ func MustClone(query *Query) *Query {
if query != nil {
q.PageNumber = query.PageNumber
q.PageSize = query.PageSize
q.Sorting = query.Sorting
q.Sorts = query.Sorts
for k, v := range query.Keywords {
q.Keywords[k] = v
}
for _, sort := range query.Sorts {
q.Sorts = append(q.Sorts, &Sort{
Key: sort.Key,
DESC: sort.DESC,
})
}
}
return q
}
// Sort specifies the order information
type Sort struct {
Key string
DESC bool
}
// Range query
type Range struct {
Min interface{}
@ -82,3 +96,40 @@ type OrList struct {
type FuzzyMatchValue struct {
Value string
}
// NewSort creates new sort
func NewSort(key string, desc bool) *Sort {
return &Sort{
Key: key,
DESC: desc,
}
}
// NewRange creates a new range
func NewRange(min, max interface{}) *Range {
return &Range{
Min: min,
Max: max,
}
}
// NewAndList creates a new and list
func NewAndList(values ...interface{}) *AndList {
return &AndList{
Values: values,
}
}
// NewOrList creates a new or list
func NewOrList(values ...interface{}) *OrList {
return &OrList{
Values: values,
}
}
// NewFuzzyMatchValue creates a new fuzzy match
func NewFuzzyMatchValue(value string) *FuzzyMatchValue {
return &FuzzyMatchValue{
Value: value,
}
}

1
src/pkg/artifact/dao/dao.go
View File

@ -97,7 +97,6 @@ func (d *dao) List(ctx context.Context, query *q.Query) ([]*Artifact, error) {
if err != nil {
return nil, err
}
qs = qs.OrderBy("-PushTime", "ID")
if _, err = qs.All(&artifacts); err != nil {
return nil, err
}

2
src/pkg/artifact/dao/model.go
View File

@ -37,7 +37,7 @@ type Artifact struct {
Digest string `orm:"column(digest)"`
Size int64 `orm:"column(size)"`
Icon string `orm:"column(icon)"`
PushTime time.Time `orm:"column(push_time)"`
PushTime time.Time `orm:"column(push_time)" sort:"default:desc"`
PullTime time.Time `orm:"column(pull_time)"`
ExtraAttrs string `orm:"column(extra_attrs)"` // json string
Annotations string `orm:"column(annotations);type(jsonb)"` // json string

9
src/pkg/audit/dao/dao.go
View File

@ -45,13 +45,7 @@ type dao struct{}
// Count ...
func (d *dao) Count(ctx context.Context, query *q.Query) (int64, error) {
if query != nil {
// ignore the page number and size
query = &q.Query{
Keywords: query.Keywords,
}
}
qs, err := orm.QuerySetter(ctx, &model.AuditLog{}, query)
qs, err := orm.QuerySetterForCount(ctx, &model.AuditLog{}, query)
if err != nil {
return 0, err
}
@ -65,7 +59,6 @@ func (d *dao) List(ctx context.Context, query *q.Query) ([]*model.AuditLog, erro
if err != nil {
return nil, err
}
qs = qs.OrderBy("-op_time")
if _, err = qs.All(&audit); err != nil {
return nil, err
}

2
src/pkg/audit/model/model.go
View File

@ -17,7 +17,7 @@ type AuditLog struct {
ResourceType string `orm:"column(resource_type)" json:"resource_type"`
Resource string `orm:"column(resource)" json:"resource"`
Username string `orm:"column(username)" json:"username"`
OpTime time.Time `orm:"column(op_time)" json:"op_time"`
OpTime time.Time `orm:"column(op_time)" json:"op_time" sort:"default:desc"`
}
// TableName for audit log

4
src/pkg/blob/dao/dao.go
View File

@ -222,10 +222,6 @@ func (d *dao) ListBlobs(ctx context.Context, query *q.Query) ([]*models.Blob, er
if err != nil {
return nil, err
}
if query.Sorting != "" {
qs = qs.OrderBy(query.Sorting)
}
blobs := []*models.Blob{}
if _, err = qs.All(&blobs); err != nil {
return nil, err

10
src/pkg/immutable/dao/dao.go
View File

@ -100,9 +100,6 @@ func (i *iDao) ListImmutableRules(ctx context.Context, query *q.Query) ([]*model
if err != nil {
return nil, err
}
if query.Sorting != "" {
qs = qs.OrderBy(query.Sorting)
}
if _, err = qs.All(&rules); err != nil {
return nil, err
}
@ -111,12 +108,7 @@ func (i *iDao) ListImmutableRules(ctx context.Context, query *q.Query) ([]*model
// Count ...
func (i *iDao) Count(ctx context.Context, query *q.Query) (int64, error) {
query = q.MustClone(query)
query.Sorting = ""
query.PageNumber = 0
query.PageSize = 0
qs, err := orm.QuerySetter(ctx, &model.ImmutableRule{}, query)
qs, err := orm.QuerySetterForCount(ctx, &model.ImmutableRule{}, query)
if err != nil {
return 0, err
}

8
src/pkg/p2p/preheat/dao/instance/dao.go
View File

@ -123,13 +123,7 @@ func (d *dao) Delete(ctx context.Context, id int64) error {
// List count instances by query params.
func (d *dao) Count(ctx context.Context, query *q.Query) (total int64, err error) {
if query != nil {
// ignore the page number and size
query = &q.Query{
Keywords: query.Keywords,
}
}
qs, err := orm.QuerySetter(ctx, &provider.Instance{}, query)
qs, err := orm.QuerySetterForCount(ctx, &provider.Instance{}, query)
if err != nil {
return 0, err
}

12
src/pkg/p2p/preheat/dao/policy/dao.go
View File

@ -51,14 +51,7 @@ type dao struct{}
// Count returns the total count of policies according to the query
func (d *dao) Count(ctx context.Context, query *q.Query) (total int64, err error) {
if query != nil {
// ignore the page number and size
query = &q.Query{
Keywords: query.Keywords,
}
}
qs, err := orm.QuerySetter(ctx, &policy.Schema{}, query)
qs, err := orm.QuerySetterForCount(ctx, &policy.Schema{}, query)
if err != nil {
return 0, err
}
@ -172,11 +165,8 @@ func (d *dao) List(ctx context.Context, query *q.Query) (schemas []*policy.Schem
if err != nil {
return
}
qs = qs.OrderBy("UpdatedTime", "ID")
if _, err = qs.All(&schemas); err != nil {
return
}
return schemas, nil
}

2
src/pkg/p2p/preheat/models/policy/policy.go
View File

@ -73,7 +73,7 @@ type Schema struct {
TriggerStr string `orm:"column(trigger)" json:"-"`
Enabled bool `orm:"column(enabled)" json:"enabled"`
CreatedAt time.Time `orm:"column(creation_time)" json:"creation_time"`
UpdatedTime time.Time `orm:"column(update_time)" json:"update_time"`
UpdatedTime time.Time `orm:"column(update_time)" json:"update_time" sort:"default"`
}
// TableName specifies the policy schema table name.

10
src/pkg/project/dao/dao.go
View File

@ -96,11 +96,7 @@ func (d *dao) Create(ctx context.Context, project *models.Project) (int64, error
func (d *dao) Count(ctx context.Context, query *q.Query) (total int64, err error) {
query = q.MustClone(query)
query.Keywords["deleted"] = false
query.Sorting = ""
query.PageNumber = 0
query.PageSize = 0
qs, err := orm.QuerySetter(ctx, &models.Project{}, query)
qs, err := orm.QuerySetterForCount(ctx, &models.Project{}, query)
if err != nil {
return 0, err
}
@ -164,10 +160,6 @@ func (d *dao) List(ctx context.Context, query *q.Query) ([]*models.Project, erro
return nil, err
}
if query.Sorting != "" {
qs = qs.OrderBy(query.Sorting)
}
projects := []*models.Project{}
if _, err := qs.All(&projects); err != nil {
return nil, err

12
src/pkg/repository/dao/dao.go
View File

@ -17,12 +17,13 @@ package dao
import (
"context"
"fmt"
"time"
o "github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
"time"
)
// DAO is the data access object interface for repository
@ -53,13 +54,7 @@ func New() DAO {
type dao struct{}
func (d *dao) Count(ctx context.Context, query *q.Query) (int64, error) {
if query != nil {
// ignore the page number and size
query = &q.Query{
Keywords: query.Keywords,
}
}
qs, err := orm.QuerySetter(ctx, &models.RepoRecord{}, query)
qs, err := orm.QuerySetterForCount(ctx, &models.RepoRecord{}, query)
if err != nil {
return 0, err
}
@ -71,7 +66,6 @@ func (d *dao) List(ctx context.Context, query *q.Query) ([]*models.RepoRecord, e
if err != nil {
return nil, err
}
qs = qs.OrderBy("-CreationTime", "RepositoryID")
if _, err = qs.All(&repositories); err != nil {
return nil, err
}

10
src/pkg/robot/dao/dao.go
View File

@ -83,12 +83,7 @@ func (d *dao) Get(ctx context.Context, id int64) (*model.Robot, error) {
}
func (d *dao) Count(ctx context.Context, query *q.Query) (int64, error) {
query = q.MustClone(query)
query.Sorting = ""
query.PageNumber = 0
query.PageSize = 0
qs, err := orm.QuerySetter(ctx, &model.Robot{}, query)
qs, err := orm.QuerySetterForCount(ctx, &model.Robot{}, query)
if err != nil {
return 0, err
}
@ -119,9 +114,6 @@ func (d *dao) List(ctx context.Context, query *q.Query) ([]*model.Robot, error)
if err != nil {
return nil, err
}
if query.Sorting != "" {
qs = qs.OrderBy(query.Sorting)
}
if _, err = qs.All(&robots); err != nil {
return nil, err
}

4
src/pkg/robot/manager.go
View File

@ -81,9 +81,5 @@ func (m *manager) Update(ctx context.Context, r *model.Robot, props ...string) e
// List ...
func (m *manager) List(ctx context.Context, query *q.Query) ([]*model.Robot, error) {
query = q.MustClone(query)
if query.Sorting == "" {
query.Sorting = "name"
}
return m.dao.List(ctx, query)
}

2
src/pkg/robot/model/model.go
View File

@ -15,7 +15,7 @@ func init() {
// Robot holds the details of a robot.
type Robot struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
Name string `orm:"column(name)" json:"name"`
Name string `orm:"column(name)" json:"name" sort:"default"`
Description string `orm:"column(description)" json:"description"`
Secret string `orm:"column(secret)" json:"secret"`
Salt string `orm:"column(salt)" json:"-"`

7
src/pkg/scan/dao/scanner/registration.go
View File

@ -28,12 +28,7 @@ func init() {
// GetTotalOfRegistrations returns the total count of scanner registrations according to the query.
func GetTotalOfRegistrations(ctx context.Context, query *q.Query) (int64, error) {
query = q.MustClone(query)
query.Sorting = ""
query.PageNumber = 0
query.PageSize = 0
qs, err := orm.QuerySetter(ctx, &Registration{}, query)
qs, err := orm.QuerySetterForCount(ctx, &Registration{}, query)
if err != nil {
return 0, err
}

9
src/pkg/tag/dao/dao.go
View File

@ -53,13 +53,7 @@ func New() DAO {
type dao struct{}
func (d *dao) Count(ctx context.Context, query *q.Query) (int64, error) {
if query != nil {
// ignore the page number and size
query = &q.Query{
Keywords: query.Keywords,
}
}
qs, err := orm.QuerySetter(ctx, &tag.Tag{}, query)
qs, err := orm.QuerySetterForCount(ctx, &tag.Tag{}, query)
if err != nil {
return 0, err
}
@ -71,7 +65,6 @@ func (d *dao) List(ctx context.Context, query *q.Query) ([]*tag.Tag, error) {
if err != nil {
return nil, err
}
qs = qs.OrderBy("-PushTime", "ID")
if _, err = qs.All(&tags); err != nil {
return nil, err
}

2
src/pkg/tag/model/tag/model.go
View File

@ -24,6 +24,6 @@ type Tag struct {
RepositoryID int64 `orm:"column(repository_id)" json:"repository_id"` // tags are the resources of repository, one repository only contains one same name tag
ArtifactID int64 `orm:"column(artifact_id)" json:"artifact_id"` // the artifact ID that the tag attaches to, it changes when pushing a same name but different digest artifact
Name string `orm:"column(name)" json:"name"`
PushTime time.Time `orm:"column(push_time)" json:"push_time"`
PushTime time.Time `orm:"column(push_time)" json:"push_time" sort:"default:desc"`
PullTime time.Time `orm:"column(pull_time)" json:"pull_time"`
}

1
src/pkg/task/dao/execution.go
View File

@ -83,7 +83,6 @@ func (e *executionDAO) List(ctx context.Context, query *q.Query) ([]*Execution,
if err != nil {
return nil, err
}
qs = qs.OrderBy("-StartTime")
if _, err = qs.All(&executions); err != nil {
return nil, err
}

4
src/pkg/task/dao/model.go
View File

@ -37,7 +37,7 @@ type Execution struct {
StatusMessage string `orm:"column(status_message)"`
Trigger string `orm:"column(trigger)"`
ExtraAttrs string `orm:"column(extra_attrs)"` // json string
StartTime time.Time `orm:"column(start_time)"`
StartTime time.Time `orm:"column(start_time)" sort:"default:desc"`
UpdateTime time.Time `orm:"column(update_time)"`
EndTime time.Time `orm:"column(end_time)"`
Revision int64 `orm:"column(revision)"`
@ -67,7 +67,7 @@ type Task struct {
RunCount int32 `orm:"column(run_count)"`
ExtraAttrs string `orm:"column(extra_attrs)"` // json string
CreationTime time.Time `orm:"column(creation_time)"`
StartTime time.Time `orm:"column(start_time)"`
StartTime time.Time `orm:"column(start_time)" sort:"default:desc"`
UpdateTime time.Time `orm:"column(update_time)"`
EndTime time.Time `orm:"column(end_time)"`
}

1
src/pkg/task/dao/task.go
View File

@ -76,7 +76,6 @@ func (t *taskDAO) List(ctx context.Context, query *q.Query) ([]*Task, error) {
if err != nil {
return nil, err
}
qs = qs.OrderBy("-StartTime")
if _, err = qs.All(&tasks); err != nil {
return nil, err
}

4
src/pkg/user/dao/dao.go
View File

@ -45,10 +45,6 @@ func (d *dao) List(ctx context.Context, query *q.Query) ([]*models.User, error)
return nil, err
}
if query.Sorting != "" {
qs = qs.OrderBy(query.Sorting)
}
var users []*models.User
if _, err := qs.All(&users); err != nil {
return nil, err

4
src/pkg/user/manager.go
View File

@ -82,10 +82,6 @@ func (m *manager) GetByName(ctx context.Context, username string) (*models.User,
// List users according to the query
func (m *manager) List(ctx context.Context, query *q.Query) (models.Users, error) {
query = q.MustClone(query)
if query.Sorting == "" {
query.Sorting = "username"
}
excludeAdmin := true
for key := range query.Keywords {
str := strings.ToLower(key)

4
src/server/v2.0/handler/artifact.go
View File

@ -77,7 +77,7 @@ func (a *artifactAPI) ListArtifacts(ctx context.Context, params operation.ListAr
}
// set query
query, err := a.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
query, err := a.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return a.SendError(ctx, err)
}
@ -292,7 +292,7 @@ func (a *artifactAPI) ListTags(ctx context.Context, params operation.ListTagsPar
return a.SendError(ctx, err)
}
// set query
query, err := a.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
query, err := a.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return a.SendError(ctx, err)
}

2
src/server/v2.0/handler/auditlog.go
View File

@ -38,7 +38,7 @@ func (a *auditlogAPI) ListAuditLogs(ctx context.Context, params auditlog.ListAud
if !secCtx.IsAuthenticated() {
return a.SendError(ctx, errors.UnauthorizedError(nil).WithMessage(secCtx.GetUsername()))
}
query, err := a.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
query, err := a.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return a.SendError(ctx, err)
}

18
src/server/v2.0/handler/base.go
View File

@ -137,32 +137,26 @@ func (b *BaseAPI) RequireAuthenticated(ctx context.Context) error {
}
// BuildQuery builds the query model according to the query string
func (b *BaseAPI) BuildQuery(ctx context.Context, query *string, pageNumber, pageSize *int64, sorts ...*string) (*q.Query, error) {
func (b *BaseAPI) BuildQuery(ctx context.Context, query, sort *string, pageNumber, pageSize *int64) (*q.Query, error) {
var (
qs string
st string
pn int64
ps int64
)
if query != nil {
qs = *query
}
if sort != nil {
st = *sort
}
if pageNumber != nil {
pn = *pageNumber
}
if pageSize != nil {
ps = *pageSize
}
r, err := q.Build(qs, pn, ps)
if err != nil {
return nil, err
}
if len(sorts) > 0 {
r.Sorting = lib.StringValue(sorts[0])
}
return r, nil
return q.Build(qs, st, pn, ps)
}
// Links return Links based on the provided pagination information

18
src/server/v2.0/handler/base_test.go
View File

@ -38,36 +38,44 @@ func (b *baseHandlerTestSuite) SetupSuite() {
}
func (b *baseHandlerTestSuite) TestBuildQuery() {
// nil query string and pagination pointer
// nil input
var (
query *string
sort *string
pageNumber *int64
pageSize *int64
)
q, err := b.base.BuildQuery(nil, query, pageNumber, pageSize)
q, err := b.base.BuildQuery(nil, query, sort, pageNumber, pageSize)
b.Require().Nil(err)
b.Require().NotNil(q)
b.NotNil(q.Keywords)
// not nil query string and pagination pointer
// not nil input
var (
qs = "q=a=b"
st = "a,-c"
pn int64 = 1
ps int64 = 10
)
q, err = b.base.BuildQuery(nil, &qs, &pn, &ps)
q, err = b.base.BuildQuery(nil, &qs, &st, &pn, &ps)
b.Require().Nil(err)
b.Require().NotNil(q)
b.Equal(int64(1), q.PageNumber)
b.Equal(int64(10), q.PageSize)
b.NotNil(q.Keywords)
b.Require().Len(q.Sorts, 2)
b.Equal("a", q.Sorts[0].Key)
b.False(q.Sorts[0].DESC)
b.Equal("c", q.Sorts[1].Key)
b.True(q.Sorts[1].DESC)
var (
qs1 = "q=a%3Db"
st1 = ""
pn1 int64 = 1
ps1 int64 = 10
)
q, err = b.base.BuildQuery(nil, &qs1, &pn1, &ps1)
q, err = b.base.BuildQuery(nil, &qs1, &st1, &pn1, &ps1)
b.Require().Nil(err)
b.Require().NotNil(q)
b.Equal(int64(1), q.PageNumber)

2
src/server/v2.0/handler/gc.go
View File

@ -136,7 +136,7 @@ func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryP
if err := g.RequireSystemAccess(ctx, rbac.ActionList, rbac.ResourceGarbageCollection); err != nil {
return g.SendError(ctx, err)
}
query, err := g.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
query, err := g.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return g.SendError(ctx, err)
}

2
src/server/v2.0/handler/immutable.go
View File

@ -94,7 +94,7 @@ func (ia *immutableAPI) ListImmuRules(ctx context.Context, params operation.List
return ia.SendError(ctx, err)
}
query, err := ia.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
query, err := ia.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return ia.SendError(ctx, err)
}

8
src/server/v2.0/handler/preheat.go
View File

@ -119,7 +119,7 @@ func (api *preheatAPI) ListInstances(ctx context.Context, params operation.ListI
var payload []*models.Instance
query, err := api.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
query, err := api.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return api.SendError(ctx, err)
}
@ -320,7 +320,7 @@ func (api *preheatAPI) ListPolicies(ctx context.Context, params operation.ListPo
return api.SendError(ctx, err)
}
query, err := api.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
query, err := api.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return api.SendError(ctx, err)
}
@ -600,7 +600,7 @@ func (api *preheatAPI) ListExecutions(ctx context.Context, params operation.List
return api.SendError(ctx, err)
}
query, err := api.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
query, err := api.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return api.SendError(ctx, err)
}
@ -677,7 +677,7 @@ func (api *preheatAPI) ListTasks(ctx context.Context, params operation.ListTasks
return api.SendError(ctx, err)
}
query, err := api.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
query, err := api.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return api.SendError(ctx, err)
}

5
src/server/v2.0/handler/project.go
View File

@ -260,7 +260,7 @@ func (a *projectAPI) GetLogs(ctx context.Context, params operation.GetLogsParams
if err != nil {
return a.SendError(ctx, err)
}
query, err := a.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
query, err := a.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return a.SendError(ctx, err)
}
@ -379,7 +379,6 @@ func (a *projectAPI) HeadProject(ctx context.Context, params operation.HeadProje
func (a *projectAPI) ListProjects(ctx context.Context, params operation.ListProjectsParams) middleware.Responder {
query := q.New(q.KeyWords{})
query.Sorting = "name"
query.PageNumber = *params.Page
query.PageSize = *params.PageSize
@ -530,7 +529,7 @@ func (a *projectAPI) ListScannerCandidatesOfProject(ctx context.Context, params
return a.SendError(ctx, err)
}
query, err := a.BuildQuery(ctx, params.Q, params.Page, params.PageSize, params.Sort)
query, err := a.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return a.SendError(ctx, err)
}

16
src/server/v2.0/handler/quota.go
View File

@ -57,14 +57,14 @@ func (qa *quotaAPI) ListQuotas(ctx context.Context, params operation.ListQuotasP
return qa.SendError(ctx, err)
}
query := &q.Query{
Keywords: q.KeyWords{
"reference": lib.StringValue(params.Reference),
"reference_id": lib.StringValue(params.ReferenceID),
},
PageNumber: *params.Page,
PageSize: *params.PageSize,
Sorting: lib.StringValue(params.Sort),
query, err := qa.BuildQuery(ctx, nil, params.Sort, params.Page, params.PageSize)
if err != nil {
return qa.SendError(ctx, err)
}
query.Keywords = q.KeyWords{
"reference": lib.StringValue(params.Reference),
"reference_id": lib.StringValue(params.ReferenceID),
}
total, err := qa.quotaCtl.Count(ctx, query)

4
src/server/v2.0/handler/replication.go
View File

@ -99,7 +99,7 @@ func (r *replicationAPI) ListReplicationExecutions(ctx context.Context, params o
if err := r.RequireSystemAccess(ctx, rbac.ActionList, rbac.ResourceReplication); err != nil {
return r.SendError(ctx, err)
}
query, err := r.BuildQuery(ctx, nil, params.Page, params.PageSize)
query, err := r.BuildQuery(ctx, nil, params.Sort, params.Page, params.PageSize)
if err != nil {
return r.SendError(ctx, err)
}
@ -172,7 +172,7 @@ func (r *replicationAPI) ListReplicationTasks(ctx context.Context, params operat
if err := r.RequireSystemAccess(ctx, rbac.ActionList, rbac.ResourceReplication); err != nil {
return r.SendError(ctx, err)
}
query, err := r.BuildQuery(ctx, nil, params.Page, params.PageSize)
query, err := r.BuildQuery(ctx, nil, params.Sort, params.Page, params.PageSize)
if err != nil {
return r.SendError(ctx, err)
}

2
src/server/v2.0/handler/repository.go
View File

@ -66,7 +66,7 @@ func (r *repositoryAPI) ListRepositories(ctx context.Context, params operation.L
}
// set query
query, err := r.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
query, err := r.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return r.SendError(ctx, err)
}

4
src/server/v2.0/handler/retention.go
View File

@ -265,7 +265,7 @@ func (r *retentionAPI) OperateRetentionExecution(ctx context.Context, params ope
}
func (r *retentionAPI) ListRetentionExecutions(ctx context.Context, params operation.ListRetentionExecutionsParams) middleware.Responder {
query, err := r.BuildQuery(ctx, nil, params.Page, params.PageSize)
query, err := r.BuildQuery(ctx, nil, nil, params.Page, params.PageSize)
if err != nil {
return r.SendError(ctx, err)
}
@ -295,7 +295,7 @@ func (r *retentionAPI) ListRetentionExecutions(ctx context.Context, params opera
}
func (r *retentionAPI) ListRetentionTasks(ctx context.Context, params operation.ListRetentionTasksParams) middleware.Responder {
query, err := r.BuildQuery(ctx, nil, params.Page, params.PageSize)
query, err := r.BuildQuery(ctx, nil, nil, params.Page, params.PageSize)
if err != nil {
return r.SendError(ctx, err)
}

2
src/server/v2.0/handler/robot.go
View File

@ -106,7 +106,7 @@ func (rAPI *robotAPI) ListRobot(ctx context.Context, params operation.ListRobotP
return rAPI.SendError(ctx, err)
}
query, err := rAPI.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
query, err := rAPI.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return rAPI.SendError(ctx, err)
}

2
src/server/v2.0/handler/robotV1.go
View File

@ -142,7 +142,7 @@ func (rAPI *robotV1API) ListRobotV1(ctx context.Context, params operation.ListRo
return rAPI.SendError(ctx, err)
}
query, err := rAPI.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
query, err := rAPI.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return rAPI.SendError(ctx, err)
}

4
src/server/v2.0/handler/scan_all.go
View File

@ -181,7 +181,7 @@ func (s *scanAllAPI) createOrUpdateScanAllSchedule(ctx context.Context, cronType
func (s *scanAllAPI) getScanAllSchedule(ctx context.Context) (*scheduler.Schedule, error) {
query := q.New(q.KeyWords{"vendor_type": scan.VendorTypeScanAll})
schedules, err := s.scheduler.ListSchedules(ctx, query.First("-creation_time"))
schedules, err := s.scheduler.ListSchedules(ctx, query.First(q.NewSort("creation_time", true)))
if err != nil {
return nil, err
}
@ -240,7 +240,7 @@ func (s *scanAllAPI) getLatestScanAllExecution(ctx context.Context, trigger ...s
query.Keywords["trigger"] = trigger[0]
}
executions, err := s.execMgr.List(ctx, query.First("-start_time"))
executions, err := s.execMgr.List(ctx, query.First(q.NewSort("start_time", true)))
if err != nil {
return nil, err
}

2
src/server/v2.0/handler/scanner.go
View File

@ -125,7 +125,7 @@ func (s *scannerAPI) ListScanners(ctx context.Context, params operation.ListScan
return s.SendError(ctx, err)
}
query, err := s.BuildQuery(ctx, params.Q, params.Page, params.PageSize, params.Sort)
query, err := s.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return s.SendError(ctx, err)
}