Merge pull request #3517 from ywk253100/171031_filter

Add replication filter implement

src/replication/consts.go

@@ -1,10 +1,10 @@
 package replication
 
 const (
-	//Kind of filter item is 'project'
-	filterItemKindProject = "project"
-	//Kind of filter item is 'repository'
-	filterItemKindRepository = "repository"
-	//Kind of filter item is 'tag'
-	filterItemKindTag = "tag"
+	//FilterItemKindProject : Kind of filter item is 'project'
+	FilterItemKindProject = "project"
+	//FilterItemKindRepository : Kind of filter item is 'repository'
+	FilterItemKindRepository = "repository"
+	//FilterItemKindTag : Kind of filter item is 'tag'
+	FilterItemKindTag = "tag"
 )

src/replication/models/filter_config.go

@@ -3,5 +3,5 @@ package models
 //FilterConfig is data model to provide configurations to the filters.
 type FilterConfig struct {
 	//The pattern for fuzzy matching
-	pattern string
+	Pattern string
 }

src/replication/models/filter_item.go

@@ -4,16 +4,16 @@ package models
 type FilterItem struct {
 
 	//The kind of the filtering resources. Support 'project','repository' and 'tag' etc.
-	kind string
+	Kind string
 
 	//The key value of resource which can be used to filter out the resource matched with specified pattern.
 	//E.g:
 	//kind == 'project', value will be project name;
 	//kind == 'repository', value will be repository name
 	//kind == 'tag', value will be tag name.
-	value string
+	Value string
 
 	//Extension placeholder.
 	//To append more additional information if required by the filter.
-	metadata map[string]interface{}
+	Metadata map[string]interface{}
 }

src/replication/source/filter.go

@@ -7,11 +7,8 @@ import (
 //Filter define the operations of selecting the matched resources from the candidates
 //according to the specified pattern.
 type Filter interface {
-	//Initialize the filter with specified configurations like pattern definition
-	Init(config models.FilterConfig)
-
-	//Set Convertor if necessary
-	SetConvertor(convertor Convertor)
+	//Initialize the filter
+	Init() error
 
 	//Return the convertor if existing or nil if never set
 	GetConvertor() Convertor

src/replication/source/pattern_filter.go

@@ -0,0 +1,84 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package source
+
+import (
+	"regexp"
+
+	"github.com/vmware/harbor/src/common/utils/log"
+	"github.com/vmware/harbor/src/replication/models"
+)
+
+// PatternFilter implements Filter interface for pattern filter
+type PatternFilter struct {
+	kind      string
+	pattern   string
+	convertor Convertor
+}
+
+// NewPatternFilter returns an instance of PatternFilter
+func NewPatternFilter(kind, pattern string, convertor ...Convertor) *PatternFilter {
+	filer := &PatternFilter{
+		kind:    kind,
+		pattern: pattern,
+	}
+
+	if len(convertor) > 0 {
+		filer.convertor = convertor[0]
+	}
+
+	return filer
+}
+
+// Init the filter. nil implement for now
+func (p *PatternFilter) Init() error {
+	return nil
+}
+
+// GetConvertor returns the convertor
+func (p *PatternFilter) GetConvertor() Convertor {
+	return p.convertor
+}
+
+// DoFilter filters resources
+func (p *PatternFilter) DoFilter(filterItems []models.FilterItem) []models.FilterItem {
+	items := []models.FilterItem{}
+	for _, item := range filterItems {
+		if item.Kind != p.kind {
+			log.Warningf("unexpected filter item kind, expected: %s, got: %s, skip",
+				p.kind, item.Kind)
+			continue
+		}
+
+		matched, err := regexp.MatchString(p.pattern, item.Value)
+		if err != nil {
+			log.Errorf("failed to match pattern %s, value %s: %v, skip",
+				p.pattern, item.Value, err)
+			continue
+		}
+
+		if !matched {
+			log.Debugf("%s does not match to the %s filter %s, skip",
+				item.Value, p.kind, p.pattern)
+			continue
+		}
+
+		log.Debugf("add %s to the result of %s filter %s",
+			item.Value, p.kind, p.pattern)
+		items = append(items, item)
+	}
+
+	return items
+}

src/replication/source/pattern_filter_test.go

@@ -0,0 +1,63 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package source
+
+import (
+	"github.com/stretchr/testify/assert"
+	"github.com/vmware/harbor/src/replication"
+	"github.com/vmware/harbor/src/replication/models"
+
+	"testing"
+)
+
+var pfilter = NewPatternFilter(replication.FilterItemKindTag, "library/ubuntu:release-*", nil)
+
+func TestPatternFilterInit(t *testing.T) {
+	assert.Nil(t, pfilter.Init())
+}
+
+func TestPatternFilterGetConvertor(t *testing.T) {
+	assert.Nil(t, pfilter.GetConvertor())
+}
+
+func TestPatternFilterDoFilter(t *testing.T) {
+	items := []models.FilterItem{
+		models.FilterItem{
+			Kind: replication.FilterItemKindProject,
+		},
+		models.FilterItem{
+			Kind: replication.FilterItemKindRepository,
+		},
+		models.FilterItem{
+			Kind:  replication.FilterItemKindTag,
+			Value: "library/ubuntu:release-14.04",
+		},
+		models.FilterItem{
+			Kind:  replication.FilterItemKindTag,
+			Value: "library/ubuntu:release-16.04",
+		},
+		models.FilterItem{
+			Kind:  replication.FilterItemKindTag,
+			Value: "library/ubuntu:test",
+		},
+	}
+	result := pfilter.DoFilter(items)
+	assert.Equal(t, 2, len(result))
+	assert.Equal(t, replication.FilterItemKindTag, result[0].Kind)
+	assert.Equal(t, "library/ubuntu:release-14.04", result[0].Value)
+	assert.Equal(t, replication.FilterItemKindTag, result[1].Kind)
+	assert.Equal(t, "library/ubuntu:release-16.04", result[1].Value)
+
+}

src/replication/source/tag_combination_filter.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package source
+
+import (
+	"strings"
+
+	"github.com/vmware/harbor/src/common/utils/log"
+	"github.com/vmware/harbor/src/replication"
+	"github.com/vmware/harbor/src/replication/models"
+)
+
+// TagCombinationFilter implements Filter interface for merging tag filter items
+// whose repository are same into one repository filter item
+type TagCombinationFilter struct{}
+
+// NewTagCombinationFilter returns an instance of TagCombinationFilter
+func NewTagCombinationFilter() *TagCombinationFilter {
+	return &TagCombinationFilter{}
+}
+
+// Init the filter. nil implement for now
+func (t *TagCombinationFilter) Init() error {
+	return nil
+}
+
+// GetConvertor returns the convertor
+func (t *TagCombinationFilter) GetConvertor() Convertor {
+	return nil
+}
+
+// DoFilter filters resources
+func (t *TagCombinationFilter) DoFilter(filterItems []models.FilterItem) []models.FilterItem {
+	repos := map[string][]string{}
+	for _, item := range filterItems {
+		if item.Kind != replication.FilterItemKindTag {
+			log.Warningf("unexpected filter item kind, expected: %s, got: %s, skip",
+				replication.FilterItemKindTag, item.Kind)
+			continue
+		}
+
+		strs := strings.Split(item.Value, ":")
+		if len(strs) != 2 {
+			log.Warningf("unexpected image format: %s, skip", item.Value)
+			continue
+		}
+
+		repos[strs[0]] = append(repos[strs[0]], strs[1])
+	}
+
+	items := []models.FilterItem{}
+	for repo, tags := range repos {
+		items = append(items, models.FilterItem{
+			Kind:  replication.FilterItemKindRepository,
+			Value: repo,
+			Metadata: map[string]interface{}{
+				"tags": tags,
+			},
+		})
+	}
+
+	return items
+}

src/replication/source/tag_combination_filter_test.go

@@ -0,0 +1,83 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package source
+
+import (
+	"github.com/stretchr/testify/assert"
+	"github.com/vmware/harbor/src/replication"
+	"github.com/vmware/harbor/src/replication/models"
+
+	"testing"
+)
+
+var tcfilter = NewTagCombinationFilter()
+
+func TestTagCombinationFilteInit(t *testing.T) {
+	assert.Nil(t, tcfilter.Init())
+}
+
+func TestTagCombinationFilteGetConvertor(t *testing.T) {
+	assert.Nil(t, tcfilter.GetConvertor())
+}
+
+func TestTagCombinationFilteDoFilter(t *testing.T) {
+	items := []models.FilterItem{
+		models.FilterItem{
+			Kind: replication.FilterItemKindProject,
+		},
+		models.FilterItem{
+			Kind: replication.FilterItemKindRepository,
+		},
+		models.FilterItem{
+			Kind:  replication.FilterItemKindTag,
+			Value: "library/ubuntu:invalid_tag:latest",
+		},
+		models.FilterItem{
+			Kind:  replication.FilterItemKindTag,
+			Value: "library/ubuntu:14.04",
+		},
+		models.FilterItem{
+			Kind:  replication.FilterItemKindTag,
+			Value: "library/ubuntu:16.04",
+		},
+		models.FilterItem{
+			Kind:  replication.FilterItemKindTag,
+			Value: "library/centos:7",
+		},
+	}
+	result := tcfilter.DoFilter(items)
+	assert.Equal(t, 2, len(result))
+
+	var ubuntu, centos models.FilterItem
+	if result[0].Value == "library/ubuntu" {
+		ubuntu = result[0]
+		centos = result[1]
+	} else {
+		centos = result[0]
+		ubuntu = result[1]
+	}
+
+	assert.Equal(t, replication.FilterItemKindRepository, ubuntu.Kind)
+	assert.Equal(t, "library/ubuntu", ubuntu.Value)
+	metadata, ok := ubuntu.Metadata["tags"].([]string)
+	assert.True(t, ok)
+	assert.EqualValues(t, []string{"14.04", "16.04"}, metadata)
+
+	assert.Equal(t, replication.FilterItemKindRepository, centos.Kind)
+	assert.Equal(t, "library/centos", centos.Value)
+	metadata, ok = centos.Metadata["tags"].([]string)
+	assert.True(t, ok)
+	assert.EqualValues(t, []string{"7"}, metadata)
+}