Merge pull request #12383 from ywk253100/200702_registry_api

Suport filtering registries by type in listing registry API

api/v2.0/legacy_swagger.yaml

@@ -1458,13 +1458,14 @@ paths:
     get:
       summary: List registries.
       description: |
-        This endpoint let user list filtered registries by name, if name is nil, list returns all registries.
+        List registries according to the query.
       parameters:
         - name: name
           in: query
           type: string
           required: false
-          description: Registry's name.
+          description: Deprecated, use `q` instead.
+        - $ref: '#/parameters/query'
       tags:
         - Products
       responses:
@@ -5668,3 +5669,10 @@ definitions:
     type: string
     description: Webhook supportted notify type.
     example: 'http'
+parameters:
+  query:
+    name: q
+    description: Query string to query resources. Supported query patterns are "exact match(k=v)", "fuzzy match(k=~v)", "range(k=[min~max])", "list with union releationship(k={v1 v2 v3})" and "list with intersetion relationship(k=(v1 v2 v3))". The value of range and list can be string(enclosed by " or '), integer or time(in format "2020-04-09 02:36:00"). All of these query patterns should be put in the query string "q=xxx" and splitted by ",". e.g. q=k1=v1,k2=~v2,k3=[min~max]
+    in: query
+    type: string
+    required: false

make/photon/prepare/templates/core/env.jinja

@@ -47,6 +47,7 @@ WITH_CHARTMUSEUM={{with_chartmuseum}}
 REGISTRY_CREDENTIAL_USERNAME={{registry_username}}
 REGISTRY_CREDENTIAL_PASSWORD={{registry_password}}
 CSRF_KEY={{csrf_key}}
+PERMITTED_REGISTRY_TYPES_FOR_PROXY_CACHE=docker-hub,harbor
 
 HTTP_PROXY={{core_http_proxy}}
 HTTPS_PROXY={{core_https_proxy}}

src/controller/event/handler/webhook/artifact/replication_test.go

@@ -5,6 +5,7 @@ import (
 	"github.com/goharbor/harbor/src/controller/event"
 	"github.com/goharbor/harbor/src/core/config"
 	"github.com/goharbor/harbor/src/core/promgr/metamgr"
+	"github.com/goharbor/harbor/src/lib/q"
 	"github.com/goharbor/harbor/src/pkg/notification"
 	"github.com/goharbor/harbor/src/replication"
 	daoModels "github.com/goharbor/harbor/src/replication/dao/models"
@@ -158,7 +159,7 @@ func (f *fakedReplicationRegistryMgr) Add(*model.Registry) (int64, error) {
 }
 
 // List registries, returns total count, registry list and error
-func (f *fakedReplicationRegistryMgr) List(...*model.RegistryQuery) (int64, []*model.Registry, error) {
+func (f *fakedReplicationRegistryMgr) List(query *q.Query) (int64, []*model.Registry, error) {
 	return 0, nil, nil
 }
 

src/core/api/project.go

@@ -143,6 +143,17 @@ func (p *ProjectAPI) Post() {
 			p.SendNotFoundError(fmt.Errorf("registry %d not found", pro.RegistryID))
 			return
 		}
+		permitted := false
+		for _, t := range config.GetPermittedRegistryTypesForProxyCache() {
+			if string(registry.Type) == t {
+				permitted = true
+				break
+			}
+		}
+		if !permitted {
+			p.SendBadRequestError(fmt.Errorf("unsupported registry type %s", string(registry.Type)))
+			return
+		}
 	}
 
 	var hardLimits types.ResourceList

src/core/api/registry.go

@@ -11,6 +11,7 @@ import (
 	"github.com/goharbor/harbor/src/common/utils"
 	"github.com/goharbor/harbor/src/core/api/models"
 	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/lib/q"
 	"github.com/goharbor/harbor/src/replication"
 	"github.com/goharbor/harbor/src/replication/adapter"
 	"github.com/goharbor/harbor/src/replication/event"
@@ -165,15 +166,24 @@ func hideAccessSecret(credential *model.Credential) {
 	credential.AccessSecret = "*****"
 }
 
-// List lists all registries that match a given registry name.
+// List lists all registries
 func (t *RegistryAPI) List() {
-	name := t.GetString("name")
+	queryStr := t.GetString("q")
-
+	// keep backward compatibility for the "name" query
-	_, registries, err := t.manager.List(&model.RegistryQuery{
+	if len(queryStr) == 0 {
-		Name: name,
+		name := t.GetString("name")
-	})
+		if len(name) > 0 {
+			queryStr = fmt.Sprintf("name=~%s", name)
+		}
+	}
+	query, err := q.Build(queryStr, 0, 0)
+	if err != nil {
+		t.SendError(err)
+		return
+	}
+
+	_, registries, err := t.manager.List(query)
 	if err != nil {
-		log.Errorf("failed to list registries %s: %v", name, err)
 		t.SendInternalServerError(err)
 		return
 	}

src/core/api/replication_policy_test.go

@@ -15,6 +15,7 @@
 package api
 
 import (
+	"github.com/goharbor/harbor/src/lib/q"
 	"net/http"
 	"testing"
 
@@ -29,7 +30,7 @@ type fakedRegistryManager struct{}
 func (f *fakedRegistryManager) Add(*model.Registry) (int64, error) {
 	return 0, nil
 }
-func (f *fakedRegistryManager) List(...*model.RegistryQuery) (int64, []*model.Registry, error) {
+func (f *fakedRegistryManager) List(query *q.Query) (int64, []*model.Registry, error) {
 	return 0, nil, nil
 }
 func (f *fakedRegistryManager) Get(id int64) (*model.Registry, error) {

src/core/config/config.go

@@ -480,3 +480,12 @@ func QuotaSetting() (*models.QuotaSetting, error) {
 		StoragePerProject: cfgMgr.Get(common.StoragePerProject).GetInt64(),
 	}, nil
 }
+
+// GetPermittedRegistryTypesForProxyCache returns the permitted registry types for proxy cache
+func GetPermittedRegistryTypesForProxyCache() []string {
+	types := os.Getenv("PERMITTED_REGISTRY_TYPES_FOR_PROXY_CACHE")
+	if len(types) == 0 {
+		return []string{}
+	}
+	return strings.Split(types, ",")
+}

src/replication/dao/registry.go

@@ -1,21 +1,15 @@
 package dao
 
 import (
+	"context"
+
 	"github.com/astaxie/beego/orm"
 	"github.com/goharbor/harbor/src/common/dao"
+	liborm "github.com/goharbor/harbor/src/lib/orm"
+	"github.com/goharbor/harbor/src/lib/q"
 	"github.com/goharbor/harbor/src/replication/dao/models"
 )
 
-// ListRegistryQuery defines the query conditions to list registry.
-type ListRegistryQuery struct {
-	// Query is name query
-	Query string
-	// Offset specifies the offset in the registry list to return
-	Offset int64
-	// Limit specifies the maximum registries to return
-	Limit int64
-}
-
 // AddRegistry add a new registry
 func AddRegistry(registry *models.Registry) (int64, error) {
 	o := dao.GetOrmer()
@@ -55,34 +49,37 @@ func GetRegistryByURL(url string) (*models.Registry, error) {
 	return &r, err
 }
 
-// ListRegistries lists registries. Registries returned are sorted by creation time.
+// ListRegistries lists registries
-// - query: query to the registry name, name query and pagination are defined.
+func ListRegistries(ctx context.Context, query *q.Query) (int64, []*models.Registry, error) {
-func ListRegistries(query ...*ListRegistryQuery) (int64, []*models.Registry, error) {
+	var countQuery *q.Query
-	o := dao.GetOrmer()
+	if query != nil {
-
+		// ignore the page number and size
-	q := o.QueryTable(&models.Registry{})
+		countQuery = &q.Query{
-	if len(query) > 0 && len(query[0].Query) > 0 {
+			Keywords: query.Keywords,
-		q = q.Filter("name__contains", query[0].Query)
+		}
 	}
-
+	countQs, err := liborm.QuerySetter(ctx, &models.Registry{}, countQuery)
-	total, err := q.Count()
 	if err != nil {
-		return -1, nil, err
+		return 0, nil, err
+	}
+	count, err := countQs.Count()
+	if err != nil {
+		return 0, nil, err
 	}
 
-	// limit being -1 means no pagination specified.
+	qs, err := liborm.QuerySetter(ctx, &models.Registry{}, query)
-	if len(query) > 0 && query[0].Limit != -1 {
+	if err != nil {
-		q = q.Offset(query[0].Offset).Limit(query[0].Limit)
+		return 0, nil, err
 	}
 	var registries []*models.Registry
-	_, err = q.All(&registries)
+	_, err = qs.All(&registries)
 	if err != nil {
-		return total, nil, err
+		return 0, nil, err
 	}
 	if registries == nil {
 		registries = []*models.Registry{}
 	}
-	return total, registries, nil
+	return count, registries, nil
 }
 
 // UpdateRegistry updates one registry

src/replication/dao/registry_test.go

@@ -3,6 +3,8 @@ package dao
 import (
 	"testing"
 
+	"github.com/goharbor/harbor/src/lib/orm"
+	"github.com/goharbor/harbor/src/lib/q"
 	"github.com/goharbor/harbor/src/replication/dao/models"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
@@ -93,62 +95,25 @@ func (suite *RegistrySuite) TestGetRegistryByURL() {
 }
 
 func (suite *RegistrySuite) TestListRegistries() {
-	assert := assert.New(suite.T())
+	ctx := orm.Context()
 
-	// Insert on more registry
+	// nil query
-	id, err := AddRegistry(testRegistry1)
+	count, registries, err := ListRegistries(ctx, nil)
-	assert.Nil(err)
+	suite.Require().Nil(err)
-	assert.NotEqual(0, id)
+	if count < 1 {
-
+		suite.T().Errorf("At least %d should be found in total, but got %d", 1, count)
-	// List all registries, should succeed
-	total, registries, err := ListRegistries()
-	assert.Nil(err)
-	if total < 2 {
-		suite.T().Errorf("At least %d should be found in total, but got %d", 2, total)
 	}
 
-	// List default registry by normal query, should succeed
+	// query by name
-	total, registries, err = ListRegistries(&ListRegistryQuery{
+	count, registries, err = ListRegistries(ctx, &q.Query{
-		Query:  "Default",
+		Keywords: map[string]interface{}{
-		Offset: 0,
+			"Name": "daoTestDefault",
-		Limit:  10,
+		},
 	})
-	assert.Nil(err)
+	suite.Require().Nil(err)
-	assert.Equal(int64(1), total)
+	suite.Require().Equal(int64(1), count)
-	assert.Equal(defaultRegistry.Name, registries[0].Name)
+	suite.Assert().Equal("daoTestDefault", registries[0].Name)
 
-	// List registry and limit to 1, should return one
-	total, registries, err = ListRegistries(&ListRegistryQuery{
-		Query:  "dao",
-		Offset: 0,
-		Limit:  1,
-	})
-	assert.Nil(err)
-	assert.Equal(int64(2), total)
-	assert.Equal(1, len(registries))
-
-	// List registry and limit set to -1, should return all
-	total, registries, err = ListRegistries(&ListRegistryQuery{
-		Limit: -1,
-	})
-	assert.Nil(err)
-	if total < 2 {
-		suite.T().Errorf("At least %d should be found in total, but got %d", 2, total)
-	}
-	if len(registries) < 2 {
-		suite.T().Errorf("At least %d should be returned, but got %d", 2, len(registries))
-	}
-
-	// List registry and large offset, should return empty
-	total, registries, err = ListRegistries(&ListRegistryQuery{
-		Offset: 10,
-		Limit:  1,
-	})
-	assert.Nil(err)
-	if total < 2 {
-		suite.T().Errorf("At least %d should be found in total, but got %d", 2, total)
-	}
-	assert.Equal(0, len(registries))
 }
 
 func (suite *RegistrySuite) TestUpdate() {

src/replication/event/handler_test.go

@@ -15,6 +15,7 @@
 package event
 
 import (
+	"github.com/goharbor/harbor/src/lib/q"
 	"testing"
 
 	"github.com/goharbor/harbor/src/replication/config"
@@ -184,7 +185,7 @@ type fakedRegistryManager struct{}
 func (f *fakedRegistryManager) Add(*model.Registry) (int64, error) {
 	return 0, nil
 }
-func (f *fakedRegistryManager) List(...*model.RegistryQuery) (int64, []*model.Registry, error) {
+func (f *fakedRegistryManager) List(query *q.Query) (int64, []*model.Registry, error) {
 	return 0, nil, nil
 }
 func (f *fakedRegistryManager) Get(id int64) (*model.Registry, error) {

src/replication/model/registry.go

@@ -16,8 +16,6 @@ package model
 
 import (
 	"time"
-
-	"github.com/goharbor/harbor/src/common/models"
 )
 
 // const definition
@@ -101,14 +99,6 @@ type Registry struct {
 	UpdateTime      time.Time   `json:"update_time"`
 }
 
-// RegistryQuery defines the query conditions for listing registries
-type RegistryQuery struct {
-	// Name is name of the registry to query
-	Name string
-	// Pagination specifies the pagination
-	Pagination *models.Pagination
-}
-
 // FilterStyle ...
 type FilterStyle struct {
 	Type   FilterType `json:"type"`

src/replication/registry/manager.go

@@ -19,6 +19,8 @@ import (
 
 	"github.com/goharbor/harbor/src/common/utils"
 	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/lib/orm"
+	"github.com/goharbor/harbor/src/lib/q"
 	"github.com/goharbor/harbor/src/replication/adapter"
 	"github.com/goharbor/harbor/src/replication/config"
 	"github.com/goharbor/harbor/src/replication/dao"
@@ -31,7 +33,7 @@ type Manager interface {
 	// Add new registry
 	Add(*model.Registry) (int64, error)
 	// List registries, returns total count, registry list and error
-	List(...*model.RegistryQuery) (int64, []*model.Registry, error)
+	List(*q.Query) (int64, []*model.Registry, error)
 	// Get the specified registry
 	Get(int64) (*model.Registry, error)
 	// GetByName gets registry by name
@@ -85,36 +87,22 @@ func (m *DefaultManager) GetByName(name string) (*model.Registry, error) {
 }
 
 // List lists registries according to query provided.
-func (m *DefaultManager) List(query ...*model.RegistryQuery) (int64, []*model.Registry, error) {
+func (m *DefaultManager) List(query *q.Query) (int64, []*model.Registry, error) {
-	var registryQueries []*dao.ListRegistryQuery
+	count, registries, err := dao.ListRegistries(orm.Context(), query)
-	if len(query) > 0 {
-		// limit being -1 indicates no pagination specified, result in all registries matching name returned.
-		listQuery := &dao.ListRegistryQuery{
-			Query: query[0].Name,
-			Limit: -1,
-		}
-		if query[0].Pagination != nil {
-			listQuery.Offset = query[0].Pagination.Page * query[0].Pagination.Size
-			listQuery.Limit = query[0].Pagination.Size
-		}
-
-		registryQueries = append(registryQueries, listQuery)
-	}
-	total, registries, err := dao.ListRegistries(registryQueries...)
 	if err != nil {
-		return -1, nil, err
+		return 0, nil, err
 	}
 
 	var results []*model.Registry
 	for _, r := range registries {
 		registry, err := fromDaoModel(r)
 		if err != nil {
-			return -1, nil, err
+			return 0, nil, err
 		}
 		results = append(results, registry)
 	}
 
-	return total, results, nil
+	return count, results, nil
 }
 
 // Add adds a new registry
@@ -158,7 +146,7 @@ func (m *DefaultManager) Remove(id int64) error {
 // HealthCheck checks health status of every registries and update their status. It will check whether a registry
 // is reachable and the credential is valid
 func (m *DefaultManager) HealthCheck() error {
-	_, registries, err := m.List()
+	_, registries, err := m.List(nil)
 	if err != nil {
 		return err
 	}