From 82b400c049d11b02f93bc43b1ee0600eb026da85 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 1 Nov 2017 14:13:34 +0800 Subject: [PATCH] Add replication filter implement --- src/replication/consts.go | 12 +-- src/replication/models/filter_config.go | 2 +- src/replication/models/filter_item.go | 6 +- src/replication/source/filter.go | 7 +- src/replication/source/pattern_filter.go | 84 +++++++++++++++++++ src/replication/source/pattern_filter_test.go | 63 ++++++++++++++ .../source/tag_combination_filter.go | 75 +++++++++++++++++ .../source/tag_combination_filter_test.go | 83 ++++++++++++++++++ 8 files changed, 317 insertions(+), 15 deletions(-) create mode 100644 src/replication/source/pattern_filter.go create mode 100644 src/replication/source/pattern_filter_test.go create mode 100644 src/replication/source/tag_combination_filter.go create mode 100644 src/replication/source/tag_combination_filter_test.go diff --git a/src/replication/consts.go b/src/replication/consts.go index 10075bcb3..76926f224 100644 --- a/src/replication/consts.go +++ b/src/replication/consts.go @@ -1,10 +1,10 @@ package replication const ( - //Kind of filter item is 'project' - filterItemKindProject = "project" - //Kind of filter item is 'repository' - filterItemKindRepository = "repository" - //Kind of filter item is 'tag' - filterItemKindTag = "tag" + //FilterItemKindProject : Kind of filter item is 'project' + FilterItemKindProject = "project" + //FilterItemKindRepository : Kind of filter item is 'repository' + FilterItemKindRepository = "repository" + //FilterItemKindTag : Kind of filter item is 'tag' + FilterItemKindTag = "tag" ) diff --git a/src/replication/models/filter_config.go b/src/replication/models/filter_config.go index cdc8e60a9..149780eaa 100644 --- a/src/replication/models/filter_config.go +++ b/src/replication/models/filter_config.go @@ -3,5 +3,5 @@ package models //FilterConfig is data model to provide configurations to the filters. type FilterConfig struct { //The pattern for fuzzy matching - pattern string + Pattern string } diff --git a/src/replication/models/filter_item.go b/src/replication/models/filter_item.go index 9565ac447..038287c78 100644 --- a/src/replication/models/filter_item.go +++ b/src/replication/models/filter_item.go @@ -4,16 +4,16 @@ package models type FilterItem struct { //The kind of the filtering resources. Support 'project','repository' and 'tag' etc. - kind string + Kind string //The key value of resource which can be used to filter out the resource matched with specified pattern. //E.g: //kind == 'project', value will be project name; //kind == 'repository', value will be repository name //kind == 'tag', value will be tag name. - value string + Value string //Extension placeholder. //To append more additional information if required by the filter. - metadata map[string]interface{} + Metadata map[string]interface{} } diff --git a/src/replication/source/filter.go b/src/replication/source/filter.go index 831e4b017..eb7b0623c 100644 --- a/src/replication/source/filter.go +++ b/src/replication/source/filter.go @@ -7,11 +7,8 @@ import ( //Filter define the operations of selecting the matched resources from the candidates //according to the specified pattern. type Filter interface { - //Initialize the filter with specified configurations like pattern definition - Init(config models.FilterConfig) - - //Set Convertor if necessary - SetConvertor(convertor Convertor) + //Initialize the filter + Init() error //Return the convertor if existing or nil if never set GetConvertor() Convertor diff --git a/src/replication/source/pattern_filter.go b/src/replication/source/pattern_filter.go new file mode 100644 index 000000000..6c895d0eb --- /dev/null +++ b/src/replication/source/pattern_filter.go @@ -0,0 +1,84 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "regexp" + + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/replication/models" +) + +// PatternFilter implements Filter interface for pattern filter +type PatternFilter struct { + kind string + pattern string + convertor Convertor +} + +// NewPatternFilter returns an instance of PatternFilter +func NewPatternFilter(kind, pattern string, convertor ...Convertor) *PatternFilter { + filer := &PatternFilter{ + kind: kind, + pattern: pattern, + } + + if len(convertor) > 0 { + filer.convertor = convertor[0] + } + + return filer +} + +// Init the filter. nil implement for now +func (p *PatternFilter) Init() error { + return nil +} + +// GetConvertor returns the convertor +func (p *PatternFilter) GetConvertor() Convertor { + return p.convertor +} + +// DoFilter filters resources +func (p *PatternFilter) DoFilter(filterItems []models.FilterItem) []models.FilterItem { + items := []models.FilterItem{} + for _, item := range filterItems { + if item.Kind != p.kind { + log.Warningf("unexpected filter item kind, expected: %s, got: %s, skip", + p.kind, item.Kind) + continue + } + + matched, err := regexp.MatchString(p.pattern, item.Value) + if err != nil { + log.Errorf("failed to match pattern %s, value %s: %v, skip", + p.pattern, item.Value, err) + continue + } + + if !matched { + log.Debugf("%s does not match to the %s filter %s, skip", + item.Value, p.kind, p.pattern) + continue + } + + log.Debugf("add %s to the result of %s filter %s", + item.Value, p.kind, p.pattern) + items = append(items, item) + } + + return items +} diff --git a/src/replication/source/pattern_filter_test.go b/src/replication/source/pattern_filter_test.go new file mode 100644 index 000000000..2f1ea372e --- /dev/null +++ b/src/replication/source/pattern_filter_test.go @@ -0,0 +1,63 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" + + "testing" +) + +var pfilter = NewPatternFilter(replication.FilterItemKindTag, "library/ubuntu:release-*", nil) + +func TestPatternFilterInit(t *testing.T) { + assert.Nil(t, pfilter.Init()) +} + +func TestPatternFilterGetConvertor(t *testing.T) { + assert.Nil(t, pfilter.GetConvertor()) +} + +func TestPatternFilterDoFilter(t *testing.T) { + items := []models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindProject, + }, + models.FilterItem{ + Kind: replication.FilterItemKindRepository, + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/ubuntu:release-14.04", + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/ubuntu:release-16.04", + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/ubuntu:test", + }, + } + result := pfilter.DoFilter(items) + assert.Equal(t, 2, len(result)) + assert.Equal(t, replication.FilterItemKindTag, result[0].Kind) + assert.Equal(t, "library/ubuntu:release-14.04", result[0].Value) + assert.Equal(t, replication.FilterItemKindTag, result[1].Kind) + assert.Equal(t, "library/ubuntu:release-16.04", result[1].Value) + +} diff --git a/src/replication/source/tag_combination_filter.go b/src/replication/source/tag_combination_filter.go new file mode 100644 index 000000000..ced506e44 --- /dev/null +++ b/src/replication/source/tag_combination_filter.go @@ -0,0 +1,75 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "strings" + + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" +) + +// TagCombinationFilter implements Filter interface for merging tag filter items +// whose repository are same into one repository filter item +type TagCombinationFilter struct{} + +// NewTagCombinationFilter returns an instance of TagCombinationFilter +func NewTagCombinationFilter() *TagCombinationFilter { + return &TagCombinationFilter{} +} + +// Init the filter. nil implement for now +func (t *TagCombinationFilter) Init() error { + return nil +} + +// GetConvertor returns the convertor +func (t *TagCombinationFilter) GetConvertor() Convertor { + return nil +} + +// DoFilter filters resources +func (t *TagCombinationFilter) DoFilter(filterItems []models.FilterItem) []models.FilterItem { + repos := map[string][]string{} + for _, item := range filterItems { + if item.Kind != replication.FilterItemKindTag { + log.Warningf("unexpected filter item kind, expected: %s, got: %s, skip", + replication.FilterItemKindTag, item.Kind) + continue + } + + strs := strings.Split(item.Value, ":") + if len(strs) != 2 { + log.Warningf("unexpected image format: %s, skip", item.Value) + continue + } + + repos[strs[0]] = append(repos[strs[0]], strs[1]) + } + + items := []models.FilterItem{} + for repo, tags := range repos { + items = append(items, models.FilterItem{ + Kind: replication.FilterItemKindRepository, + Value: repo, + Metadata: map[string]interface{}{ + "tags": tags, + }, + }) + } + + return items +} diff --git a/src/replication/source/tag_combination_filter_test.go b/src/replication/source/tag_combination_filter_test.go new file mode 100644 index 000000000..c3bfefff8 --- /dev/null +++ b/src/replication/source/tag_combination_filter_test.go @@ -0,0 +1,83 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" + + "testing" +) + +var tcfilter = NewTagCombinationFilter() + +func TestTagCombinationFilteInit(t *testing.T) { + assert.Nil(t, tcfilter.Init()) +} + +func TestTagCombinationFilteGetConvertor(t *testing.T) { + assert.Nil(t, tcfilter.GetConvertor()) +} + +func TestTagCombinationFilteDoFilter(t *testing.T) { + items := []models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindProject, + }, + models.FilterItem{ + Kind: replication.FilterItemKindRepository, + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/ubuntu:invalid_tag:latest", + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/ubuntu:14.04", + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/ubuntu:16.04", + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/centos:7", + }, + } + result := tcfilter.DoFilter(items) + assert.Equal(t, 2, len(result)) + + var ubuntu, centos models.FilterItem + if result[0].Value == "library/ubuntu" { + ubuntu = result[0] + centos = result[1] + } else { + centos = result[0] + ubuntu = result[1] + } + + assert.Equal(t, replication.FilterItemKindRepository, ubuntu.Kind) + assert.Equal(t, "library/ubuntu", ubuntu.Value) + metadata, ok := ubuntu.Metadata["tags"].([]string) + assert.True(t, ok) + assert.EqualValues(t, []string{"14.04", "16.04"}, metadata) + + assert.Equal(t, replication.FilterItemKindRepository, centos.Kind) + assert.Equal(t, "library/centos", centos.Value) + metadata, ok = centos.Metadata["tags"].([]string) + assert.True(t, ok) + assert.EqualValues(t, []string{"7"}, metadata) +}