Merge pull request #4962 from ywk253100/180509_label_filter

Add label filter to filter chain

src/replication/consts.go

@@ -7,6 +7,8 @@ const (
 	FilterItemKindRepository = "repository"
 	//FilterItemKindTag : Kind of filter item is 'tag'
 	FilterItemKindTag = "tag"
+	//FilterItemKindLabel : Kind of filter item is 'label'
+	FilterItemKindLabel = "label"
 
 	//AdaptorKindHarbor : Kind of adaptor of Harbor
 	AdaptorKindHarbor = "Harbor"

src/replication/core/controller.go

@@ -259,17 +259,34 @@ func getCandidates(policy *models.ReplicationPolicy, sourcer *source.Sourcer,
 func buildFilterChain(policy *models.ReplicationPolicy, sourcer *source.Sourcer) source.FilterChain {
 	filters := []source.Filter{}
 
-	patterns := map[string]string{}
-	for _, f := range policy.Filters {
-		patterns[f.Kind] = f.Pattern
+	fm := map[string][]models.Filter{}
+	for _, filter := range policy.Filters {
+		fm[filter.Kind] = append(fm[filter.Kind], filter)
 	}
 
 	registry := sourcer.GetAdaptor(replication.AdaptorKindHarbor)
-	// only support repository and tag filter for now
+	// repository filter
+	pattern := ""
+	repoFilters := fm[replication.FilterItemKindRepository]
+	if len(repoFilters) > 0 {
+		pattern = repoFilters[0].Value.(string)
+	}
 	filters = append(filters,
-		source.NewRepositoryFilter(patterns[replication.FilterItemKindRepository], registry))
+		source.NewRepositoryFilter(pattern, registry))
+	// tag filter
+	pattern = ""
+	tagFilters := fm[replication.FilterItemKindTag]
+	if len(tagFilters) > 0 {
+		pattern = tagFilters[0].Value.(string)
+	}
 	filters = append(filters,
-		source.NewTagFilter(patterns[replication.FilterItemKindTag], registry))
+		source.NewTagFilter(pattern, registry))
+	// label filters
+	var labelID int64
+	for _, labelFilter := range fm[replication.FilterItemKindLabel] {
+		labelID = labelFilter.Value.(int64)
+		filters = append(filters, source.NewLabelFilter(labelID))
+	}
 
 	return source.NewDefaultFilterChain(filters)
 }

src/replication/core/controller_test.go

@@ -82,8 +82,8 @@ func TestGetCandidates(t *testing.T) {
 		ID: 1,
 		Filters: []models.Filter{
 			models.Filter{
-				Kind:    replication.FilterItemKindTag,
-				Pattern: "*",
+				Kind:  replication.FilterItemKindTag,
+				Value: "*",
 			},
 		},
 		Trigger: &models.Trigger{
@@ -111,12 +111,23 @@ func TestGetCandidates(t *testing.T) {
 
 	policy.Filters = []models.Filter{
 		models.Filter{
-			Kind:    replication.FilterItemKindTag,
-			Pattern: "release-*",
+			Kind:  replication.FilterItemKindTag,
+			Value: "release-*",
 		},
 	}
 	result = getCandidates(policy, sourcer, metadata)
 	assert.Equal(t, 1, len(result))
+
+	// test label filter
+	test.InitDatabaseFromEnv()
+	policy.Filters = []models.Filter{
+		models.Filter{
+			Kind:  replication.FilterItemKindLabel,
+			Value: int64(1),
+		},
+	}
+	result = getCandidates(policy, sourcer, metadata)
+	assert.Equal(t, 0, len(result))
 }
 
 func TestBuildFilterChain(t *testing.T) {
@@ -124,12 +135,18 @@ func TestBuildFilterChain(t *testing.T) {
 		ID: 1,
 		Filters: []models.Filter{
 			models.Filter{
-				Kind:    replication.FilterItemKindRepository,
-				Pattern: "*",
+				Kind:  replication.FilterItemKindRepository,
+				Value: "*",
 			},
+
 			models.Filter{
-				Kind:    replication.FilterItemKindTag,
-				Pattern: "*",
+				Kind:  replication.FilterItemKindTag,
+				Value: "*",
+			},
+
+			models.Filter{
+				Kind:  replication.FilterItemKindLabel,
+				Value: int64(1),
 			},
 		},
 	}
@@ -137,5 +154,5 @@ func TestBuildFilterChain(t *testing.T) {
 	sourcer := source.NewSourcer()
 
 	chain := buildFilterChain(policy, sourcer)
-	assert.Equal(t, 2, len(chain.Filters()))
+	assert.Equal(t, 3, len(chain.Filters()))
 }

src/replication/models/filter.go

@@ -23,8 +23,9 @@ import (
 
 // Filter is the data model represents the filter defined by user.
 type Filter struct {
-	Kind    string `json:"kind"`
-	Pattern string `json:"pattern"`
+	Kind    string      `json:"kind"`
+	Pattern string      `json:"pattern"` // deprecated, use Value instead
+	Value   interface{} `json:"value"`
 }
 
 // Valid ...

src/replication/policy/manager.go

@@ -106,6 +106,11 @@ func convertFromPersistModel(policy *persist_models.RepPolicy) (models.Replicati
 		if err := json.Unmarshal([]byte(policy.Filters), &filters); err != nil {
 			return models.ReplicationPolicy{}, err
 		}
+		for i := range filters {
+			if filters[i].Value == nil && len(filters[i].Pattern) > 0 {
+				filters[i].Value = filters[i].Pattern
+			}
+		}
 		ply.Filters = filters
 	}
 

src/replication/source/label_filter.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2018 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package source
+
+import (
+	"fmt"
+
+	"github.com/vmware/harbor/src/common"
+	"github.com/vmware/harbor/src/common/dao"
+	"github.com/vmware/harbor/src/common/utils/log"
+	"github.com/vmware/harbor/src/replication"
+	"github.com/vmware/harbor/src/replication/models"
+)
+
+// LabelFilter filter resources according to label
+type LabelFilter struct {
+	labelID int64
+}
+
+// Init ...
+func (l *LabelFilter) Init() error {
+	return nil
+}
+
+// GetConvertor ...
+func (l *LabelFilter) GetConvertor() Convertor {
+	return nil
+}
+
+// NewLabelFilter returns an instance of LabelFilter
+func NewLabelFilter(labelID int64) *LabelFilter {
+	return &LabelFilter{
+		labelID: labelID,
+	}
+}
+
+// DoFilter filter the resources according to the label
+func (l *LabelFilter) DoFilter(items []models.FilterItem) []models.FilterItem {
+	candidates := []string{}
+	for _, item := range items {
+		candidates = append(candidates, item.Value)
+	}
+	log.Debugf("label filter candidates: %v", candidates)
+	result := []models.FilterItem{}
+	for _, item := range items {
+		hasLabel, err := hasLabel(item, l.labelID)
+		if err != nil {
+			log.Errorf("failed to check the label of resouce %v: %v, skip it", item, err)
+			continue
+		}
+		if hasLabel {
+			log.Debugf("has label %d, add %s to the label filter result list", l.labelID, item.Value)
+			result = append(result, item)
+		}
+	}
+	return result
+}
+
+func hasLabel(resource models.FilterItem, labelID int64) (bool, error) {
+	rType := ""
+	switch resource.Kind {
+	case replication.FilterItemKindProject:
+		rType = common.ResourceTypeProject
+	case replication.FilterItemKindRepository:
+		rType = common.ResourceTypeRepository
+	case replication.FilterItemKindTag:
+		rType = common.ResourceTypeImage
+	default:
+		return false, fmt.Errorf("invalid resource type: %s", resource.Kind)
+	}
+	rl, err := dao.GetResourceLabel(rType, resource.Value, labelID)
+	if err != nil {
+		return false, err
+	}
+	return rl != nil, nil
+}

src/replication/source/label_filter_test.go

@@ -0,0 +1,48 @@
+// Copyright (c) 2018 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package source
+
+import (
+	"testing"
+
+	"github.com/vmware/harbor/src/common/utils/test"
+	"github.com/vmware/harbor/src/replication"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/vmware/harbor/src/replication/models"
+)
+
+func TestInitOfLabelFilter(t *testing.T) {
+	filter := NewLabelFilter(1)
+	assert.Nil(t, filter.Init())
+}
+
+func TestGetConvertorOfLabelFilter(t *testing.T) {
+	filter := NewLabelFilter(1)
+	assert.Nil(t, filter.GetConvertor())
+}
+
+func TestDoFilterOfLabelFilter(t *testing.T) {
+	test.InitDatabaseFromEnv()
+	filter := NewLabelFilter(1)
+	items := []models.FilterItem{
+		models.FilterItem{
+			Kind:  replication.FilterItemKindTag,
+			Value: "library/hello-world:latest",
+		},
+	}
+	result := filter.DoFilter(items)
+	assert.Equal(t, 0, len(result))
+}

src/replication/source/repository_filter.go

@@ -76,8 +76,8 @@ func (r *RepositoryFilter) DoFilter(items []models.FilterItem) []models.FilterIt
 			_, repository = utils.ParseRepository(repository)
 			matched, err := match(r.pattern, repository)
 			if err != nil {
-				log.Errorf("failed to match pattern %s to value %s: %v", r.pattern, repository, err)
-				break
+				log.Errorf("failed to match pattern %s to value %s: %v, skip it", r.pattern, repository, err)
+				continue
 			}
 			if matched {
 				log.Debugf("pattern %s matched, add %s to the repository filter result list", r.pattern, item.Value)

src/replication/source/tag_filter.go

@@ -71,7 +71,7 @@ func (t *TagFilter) DoFilter(items []models.FilterItem) []models.FilterItem {
 		tag := strings.SplitN(item.Value, ":", 2)[1]
 		matched, err := match(t.pattern, tag)
 		if err != nil {
-			log.Errorf("failed to match pattern %s to value %s: %v", t.pattern, tag, err)
+			log.Errorf("failed to match pattern %s to value %s: %v, skip it", t.pattern, tag, err)
 			continue
 		}