Upstream
/
harbor
mirror of https://github.com/goharbor/harbor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #12383 from ywk253100/200702_registry_api 

Suport filtering registries by type in listing registry API
This commit is contained in:
stonezdj(Daojun Zhang) 2020-07-07 14:21:54 +08:00 committed by GitHub
parent 642953b9ef 02690d1d04
commit 6f4e8150d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 103 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
api/v2.0/legacy_swagger.yaml
View File

@ -1458,13 +1458,14 @@ paths:
get:
summary: List registries.
description: |
This endpoint let user list filtered registries by name, if name is nil, list returns all registries.
List registries according to the query.
parameters:
- name: name
in: query
type: string
required: false
description: Registry's name.
description: Deprecated, use `q` instead.
- $ref: '#/parameters/query'
tags:
- Products
responses:
@ -5668,3 +5669,10 @@ definitions:
type: string
description: Webhook supportted notify type.
example: 'http'
parameters:
query:
name: q
description: Query string to query resources. Supported query patterns are "exact match(k=v)", "fuzzy match(k=~v)", "range(k=[min~max])", "list with union releationship(k={v1 v2 v3})" and "list with intersetion relationship(k=(v1 v2 v3))". The value of range and list can be string(enclosed by " or '), integer or time(in format "2020-04-09 02:36:00"). All of these query patterns should be put in the query string "q=xxx" and splitted by ",". e.g. q=k1=v1,k2=~v2,k3=[min~max]
in: query
type: string
required: false

1
make/photon/prepare/templates/core/env.jinja
View File

@ -47,6 +47,7 @@ WITH_CHARTMUSEUM={{with_chartmuseum}}
REGISTRY_CREDENTIAL_USERNAME={{registry_username}}
REGISTRY_CREDENTIAL_PASSWORD={{registry_password}}
CSRF_KEY={{csrf_key}}
PERMITTED_REGISTRY_TYPES_FOR_PROXY_CACHE=docker-hub,harbor
HTTP_PROXY={{core_http_proxy}}
HTTPS_PROXY={{core_https_proxy}}

3
src/controller/event/handler/webhook/artifact/replication_test.go
View File

@ -5,6 +5,7 @@ import (
"github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/promgr/metamgr"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/replication"
daoModels "github.com/goharbor/harbor/src/replication/dao/models"
@ -158,7 +159,7 @@ func (f *fakedReplicationRegistryMgr) Add(*model.Registry) (int64, error) {
}
// List registries, returns total count, registry list and error
func (f *fakedReplicationRegistryMgr) List(...*model.RegistryQuery) (int64, []*model.Registry, error) {
func (f *fakedReplicationRegistryMgr) List(query *q.Query) (int64, []*model.Registry, error) {
return 0, nil, nil
}

11
src/core/api/project.go
View File

@ -143,6 +143,17 @@ func (p *ProjectAPI) Post() {
p.SendNotFoundError(fmt.Errorf("registry %d not found", pro.RegistryID))
return
}
permitted := false
for _, t := range config.GetPermittedRegistryTypesForProxyCache() {
if string(registry.Type) == t {
permitted = true
break
}
}
if !permitted {
p.SendBadRequestError(fmt.Errorf("unsupported registry type %s", string(registry.Type)))
return
}
}
var hardLimits types.ResourceList

24
src/core/api/registry.go
View File

@ -11,6 +11,7 @@ import (
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/event"
@ -165,15 +166,24 @@ func hideAccessSecret(credential *model.Credential) {
credential.AccessSecret = "*****"
}
// List lists all registries that match a given registry name.
// List lists all registries
func (t *RegistryAPI) List() {
name := t.GetString("name")
_, registries, err := t.manager.List(&model.RegistryQuery{
Name: name,
})
queryStr := t.GetString("q")
// keep backward compatibility for the "name" query
if len(queryStr) == 0 {
name := t.GetString("name")
if len(name) > 0 {
queryStr = fmt.Sprintf("name=~%s", name)
}
}
query, err := q.Build(queryStr, 0, 0)
if err != nil {
t.SendError(err)
return
}
_, registries, err := t.manager.List(query)
if err != nil {
log.Errorf("failed to list registries %s: %v", name, err)
t.SendInternalServerError(err)
return
}

3
src/core/api/replication_policy_test.go
View File

@ -15,6 +15,7 @@
package api
import (
"github.com/goharbor/harbor/src/lib/q"
"net/http"
"testing"
@ -29,7 +30,7 @@ type fakedRegistryManager struct{}
func (f *fakedRegistryManager) Add(*model.Registry) (int64, error) {
return 0, nil
}
func (f *fakedRegistryManager) List(...*model.RegistryQuery) (int64, []*model.Registry, error) {
func (f *fakedRegistryManager) List(query *q.Query) (int64, []*model.Registry, error) {
return 0, nil, nil
}
func (f *fakedRegistryManager) Get(id int64) (*model.Registry, error) {

9
src/core/config/config.go
View File

@ -480,3 +480,12 @@ func QuotaSetting() (*models.QuotaSetting, error) {
StoragePerProject: cfgMgr.Get(common.StoragePerProject).GetInt64(),
}, nil
}
// GetPermittedRegistryTypesForProxyCache returns the permitted registry types for proxy cache
func GetPermittedRegistryTypesForProxyCache() []string {
types := os.Getenv("PERMITTED_REGISTRY_TYPES_FOR_PROXY_CACHE")
if len(types) == 0 {
return []string{}
}
return strings.Split(types, ",")
}

51
src/replication/dao/registry.go
View File

@ -1,21 +1,15 @@
package dao
import (
"context"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/dao"
liborm "github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/replication/dao/models"
)
// ListRegistryQuery defines the query conditions to list registry.
type ListRegistryQuery struct {
// Query is name query
Query string
// Offset specifies the offset in the registry list to return
Offset int64
// Limit specifies the maximum registries to return
Limit int64
}
// AddRegistry add a new registry
func AddRegistry(registry *models.Registry) (int64, error) {
o := dao.GetOrmer()
@ -55,34 +49,37 @@ func GetRegistryByURL(url string) (*models.Registry, error) {
return &r, err
}
// ListRegistries lists registries. Registries returned are sorted by creation time.
// - query: query to the registry name, name query and pagination are defined.
func ListRegistries(query ...*ListRegistryQuery) (int64, []*models.Registry, error) {
o := dao.GetOrmer()
q := o.QueryTable(&models.Registry{})
if len(query) > 0 && len(query[0].Query) > 0 {
q = q.Filter("name__contains", query[0].Query)
// ListRegistries lists registries
func ListRegistries(ctx context.Context, query *q.Query) (int64, []*models.Registry, error) {
var countQuery *q.Query
if query != nil {
// ignore the page number and size
countQuery = &q.Query{
Keywords: query.Keywords,
}
}
total, err := q.Count()
countQs, err := liborm.QuerySetter(ctx, &models.Registry{}, countQuery)
if err != nil {
return -1, nil, err
return 0, nil, err
}
count, err := countQs.Count()
if err != nil {
return 0, nil, err
}
// limit being -1 means no pagination specified.
if len(query) > 0 && query[0].Limit != -1 {
q = q.Offset(query[0].Offset).Limit(query[0].Limit)
qs, err := liborm.QuerySetter(ctx, &models.Registry{}, query)
if err != nil {
return 0, nil, err
}
var registries []*models.Registry
_, err = q.All(&registries)
_, err = qs.All(&registries)
if err != nil {
return total, nil, err
return 0, nil, err
}
if registries == nil {
registries = []*models.Registry{}
}
return total, registries, nil
return count, registries, nil
}
// UpdateRegistry updates one registry

67
src/replication/dao/registry_test.go
View File

@ -3,6 +3,8 @@ package dao
import (
"testing"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/replication/dao/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
@ -93,62 +95,25 @@ func (suite *RegistrySuite) TestGetRegistryByURL() {
}
func (suite *RegistrySuite) TestListRegistries() {
assert := assert.New(suite.T())
ctx := orm.Context()
// Insert on more registry
id, err := AddRegistry(testRegistry1)
assert.Nil(err)
assert.NotEqual(0, id)
// List all registries, should succeed
total, registries, err := ListRegistries()
assert.Nil(err)
if total < 2 {
suite.T().Errorf("At least %d should be found in total, but got %d", 2, total)
// nil query
count, registries, err := ListRegistries(ctx, nil)
suite.Require().Nil(err)
if count < 1 {
suite.T().Errorf("At least %d should be found in total, but got %d", 1, count)
}
// List default registry by normal query, should succeed
total, registries, err = ListRegistries(&ListRegistryQuery{
Query: "Default",
Offset: 0,
Limit: 10,
// query by name
count, registries, err = ListRegistries(ctx, &q.Query{
Keywords: map[string]interface{}{
"Name": "daoTestDefault",
},
})
assert.Nil(err)
assert.Equal(int64(1), total)
assert.Equal(defaultRegistry.Name, registries[0].Name)
suite.Require().Nil(err)
suite.Require().Equal(int64(1), count)
suite.Assert().Equal("daoTestDefault", registries[0].Name)
// List registry and limit to 1, should return one
total, registries, err = ListRegistries(&ListRegistryQuery{
Query: "dao",
Offset: 0,
Limit: 1,
})
assert.Nil(err)
assert.Equal(int64(2), total)
assert.Equal(1, len(registries))
// List registry and limit set to -1, should return all
total, registries, err = ListRegistries(&ListRegistryQuery{
Limit: -1,
})
assert.Nil(err)
if total < 2 {
suite.T().Errorf("At least %d should be found in total, but got %d", 2, total)
}
if len(registries) < 2 {
suite.T().Errorf("At least %d should be returned, but got %d", 2, len(registries))
}
// List registry and large offset, should return empty
total, registries, err = ListRegistries(&ListRegistryQuery{
Offset: 10,
Limit: 1,
})
assert.Nil(err)
if total < 2 {
suite.T().Errorf("At least %d should be found in total, but got %d", 2, total)
}
assert.Equal(0, len(registries))
}
func (suite *RegistrySuite) TestUpdate() {

3
src/replication/event/handler_test.go
View File

@ -15,6 +15,7 @@
package event
import (
"github.com/goharbor/harbor/src/lib/q"
"testing"
"github.com/goharbor/harbor/src/replication/config"
@ -184,7 +185,7 @@ type fakedRegistryManager struct{}
func (f *fakedRegistryManager) Add(*model.Registry) (int64, error) {
return 0, nil
}
func (f *fakedRegistryManager) List(...*model.RegistryQuery) (int64, []*model.Registry, error) {
func (f *fakedRegistryManager) List(query *q.Query) (int64, []*model.Registry, error) {
return 0, nil, nil
}
func (f *fakedRegistryManager) Get(id int64) (*model.Registry, error) {

10
src/replication/model/registry.go
View File

@ -16,8 +16,6 @@ package model
import (
"time"
"github.com/goharbor/harbor/src/common/models"
)
// const definition
@ -101,14 +99,6 @@ type Registry struct {
UpdateTime time.Time `json:"update_time"`
}
// RegistryQuery defines the query conditions for listing registries
type RegistryQuery struct {
// Name is name of the registry to query
Name string
// Pagination specifies the pagination
Pagination *models.Pagination
}
// FilterStyle ...
type FilterStyle struct {
Type FilterType `json:"type"`

30
src/replication/registry/manager.go
View File

@ -19,6 +19,8 @@ import (
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/config"
"github.com/goharbor/harbor/src/replication/dao"
@ -31,7 +33,7 @@ type Manager interface {
// Add new registry
Add(*model.Registry) (int64, error)
// List registries, returns total count, registry list and error
List(...*model.RegistryQuery) (int64, []*model.Registry, error)
List(*q.Query) (int64, []*model.Registry, error)
// Get the specified registry
Get(int64) (*model.Registry, error)
// GetByName gets registry by name
@ -85,36 +87,22 @@ func (m *DefaultManager) GetByName(name string) (*model.Registry, error) {
}
// List lists registries according to query provided.
func (m *DefaultManager) List(query ...*model.RegistryQuery) (int64, []*model.Registry, error) {
var registryQueries []*dao.ListRegistryQuery
if len(query) > 0 {
// limit being -1 indicates no pagination specified, result in all registries matching name returned.
listQuery := &dao.ListRegistryQuery{
Query: query[0].Name,
Limit: -1,
}
if query[0].Pagination != nil {
listQuery.Offset = query[0].Pagination.Page * query[0].Pagination.Size
listQuery.Limit = query[0].Pagination.Size
}
registryQueries = append(registryQueries, listQuery)
}
total, registries, err := dao.ListRegistries(registryQueries...)
func (m *DefaultManager) List(query *q.Query) (int64, []*model.Registry, error) {
count, registries, err := dao.ListRegistries(orm.Context(), query)
if err != nil {
return -1, nil, err
return 0, nil, err
}
var results []*model.Registry
for _, r := range registries {
registry, err := fromDaoModel(r)
if err != nil {
return -1, nil, err
return 0, nil, err
}
results = append(results, registry)
}
return total, results, nil
return count, results, nil
}
// Add adds a new registry
@ -158,7 +146,7 @@ func (m *DefaultManager) Remove(id int64) error {
// HealthCheck checks health status of every registries and update their status. It will check whether a registry
// is reachable and the credential is valid
func (m *DefaultManager) HealthCheck() error {
_, registries, err := m.List()
_, registries, err := m.List(nil)
if err != nil {
return err
}