mirror of
https://github.com/goharbor/harbor.git
synced 2024-09-18 08:41:10 +02:00
Merge pull request #3517 from ywk253100/171031_filter
Add replication filter implement
This commit is contained in:
commit
628730921b
@ -1,10 +1,10 @@
|
|||||||
package replication
|
package replication
|
||||||
|
|
||||||
const (
|
const (
|
||||||
//Kind of filter item is 'project'
|
//FilterItemKindProject : Kind of filter item is 'project'
|
||||||
filterItemKindProject = "project"
|
FilterItemKindProject = "project"
|
||||||
//Kind of filter item is 'repository'
|
//FilterItemKindRepository : Kind of filter item is 'repository'
|
||||||
filterItemKindRepository = "repository"
|
FilterItemKindRepository = "repository"
|
||||||
//Kind of filter item is 'tag'
|
//FilterItemKindTag : Kind of filter item is 'tag'
|
||||||
filterItemKindTag = "tag"
|
FilterItemKindTag = "tag"
|
||||||
)
|
)
|
||||||
|
@ -3,5 +3,5 @@ package models
|
|||||||
//FilterConfig is data model to provide configurations to the filters.
|
//FilterConfig is data model to provide configurations to the filters.
|
||||||
type FilterConfig struct {
|
type FilterConfig struct {
|
||||||
//The pattern for fuzzy matching
|
//The pattern for fuzzy matching
|
||||||
pattern string
|
Pattern string
|
||||||
}
|
}
|
||||||
|
@ -4,16 +4,16 @@ package models
|
|||||||
type FilterItem struct {
|
type FilterItem struct {
|
||||||
|
|
||||||
//The kind of the filtering resources. Support 'project','repository' and 'tag' etc.
|
//The kind of the filtering resources. Support 'project','repository' and 'tag' etc.
|
||||||
kind string
|
Kind string
|
||||||
|
|
||||||
//The key value of resource which can be used to filter out the resource matched with specified pattern.
|
//The key value of resource which can be used to filter out the resource matched with specified pattern.
|
||||||
//E.g:
|
//E.g:
|
||||||
//kind == 'project', value will be project name;
|
//kind == 'project', value will be project name;
|
||||||
//kind == 'repository', value will be repository name
|
//kind == 'repository', value will be repository name
|
||||||
//kind == 'tag', value will be tag name.
|
//kind == 'tag', value will be tag name.
|
||||||
value string
|
Value string
|
||||||
|
|
||||||
//Extension placeholder.
|
//Extension placeholder.
|
||||||
//To append more additional information if required by the filter.
|
//To append more additional information if required by the filter.
|
||||||
metadata map[string]interface{}
|
Metadata map[string]interface{}
|
||||||
}
|
}
|
||||||
|
@ -7,11 +7,8 @@ import (
|
|||||||
//Filter define the operations of selecting the matched resources from the candidates
|
//Filter define the operations of selecting the matched resources from the candidates
|
||||||
//according to the specified pattern.
|
//according to the specified pattern.
|
||||||
type Filter interface {
|
type Filter interface {
|
||||||
//Initialize the filter with specified configurations like pattern definition
|
//Initialize the filter
|
||||||
Init(config models.FilterConfig)
|
Init() error
|
||||||
|
|
||||||
//Set Convertor if necessary
|
|
||||||
SetConvertor(convertor Convertor)
|
|
||||||
|
|
||||||
//Return the convertor if existing or nil if never set
|
//Return the convertor if existing or nil if never set
|
||||||
GetConvertor() Convertor
|
GetConvertor() Convertor
|
||||||
|
84
src/replication/source/pattern_filter.go
Normal file
84
src/replication/source/pattern_filter.go
Normal file
@ -0,0 +1,84 @@
|
|||||||
|
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package source
|
||||||
|
|
||||||
|
import (
|
||||||
|
"regexp"
|
||||||
|
|
||||||
|
"github.com/vmware/harbor/src/common/utils/log"
|
||||||
|
"github.com/vmware/harbor/src/replication/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PatternFilter implements Filter interface for pattern filter
|
||||||
|
type PatternFilter struct {
|
||||||
|
kind string
|
||||||
|
pattern string
|
||||||
|
convertor Convertor
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewPatternFilter returns an instance of PatternFilter
|
||||||
|
func NewPatternFilter(kind, pattern string, convertor ...Convertor) *PatternFilter {
|
||||||
|
filer := &PatternFilter{
|
||||||
|
kind: kind,
|
||||||
|
pattern: pattern,
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(convertor) > 0 {
|
||||||
|
filer.convertor = convertor[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
return filer
|
||||||
|
}
|
||||||
|
|
||||||
|
// Init the filter. nil implement for now
|
||||||
|
func (p *PatternFilter) Init() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetConvertor returns the convertor
|
||||||
|
func (p *PatternFilter) GetConvertor() Convertor {
|
||||||
|
return p.convertor
|
||||||
|
}
|
||||||
|
|
||||||
|
// DoFilter filters resources
|
||||||
|
func (p *PatternFilter) DoFilter(filterItems []models.FilterItem) []models.FilterItem {
|
||||||
|
items := []models.FilterItem{}
|
||||||
|
for _, item := range filterItems {
|
||||||
|
if item.Kind != p.kind {
|
||||||
|
log.Warningf("unexpected filter item kind, expected: %s, got: %s, skip",
|
||||||
|
p.kind, item.Kind)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
matched, err := regexp.MatchString(p.pattern, item.Value)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("failed to match pattern %s, value %s: %v, skip",
|
||||||
|
p.pattern, item.Value, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if !matched {
|
||||||
|
log.Debugf("%s does not match to the %s filter %s, skip",
|
||||||
|
item.Value, p.kind, p.pattern)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Debugf("add %s to the result of %s filter %s",
|
||||||
|
item.Value, p.kind, p.pattern)
|
||||||
|
items = append(items, item)
|
||||||
|
}
|
||||||
|
|
||||||
|
return items
|
||||||
|
}
|
63
src/replication/source/pattern_filter_test.go
Normal file
63
src/replication/source/pattern_filter_test.go
Normal file
@ -0,0 +1,63 @@
|
|||||||
|
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package source
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/vmware/harbor/src/replication"
|
||||||
|
"github.com/vmware/harbor/src/replication/models"
|
||||||
|
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
var pfilter = NewPatternFilter(replication.FilterItemKindTag, "library/ubuntu:release-*", nil)
|
||||||
|
|
||||||
|
func TestPatternFilterInit(t *testing.T) {
|
||||||
|
assert.Nil(t, pfilter.Init())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPatternFilterGetConvertor(t *testing.T) {
|
||||||
|
assert.Nil(t, pfilter.GetConvertor())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPatternFilterDoFilter(t *testing.T) {
|
||||||
|
items := []models.FilterItem{
|
||||||
|
models.FilterItem{
|
||||||
|
Kind: replication.FilterItemKindProject,
|
||||||
|
},
|
||||||
|
models.FilterItem{
|
||||||
|
Kind: replication.FilterItemKindRepository,
|
||||||
|
},
|
||||||
|
models.FilterItem{
|
||||||
|
Kind: replication.FilterItemKindTag,
|
||||||
|
Value: "library/ubuntu:release-14.04",
|
||||||
|
},
|
||||||
|
models.FilterItem{
|
||||||
|
Kind: replication.FilterItemKindTag,
|
||||||
|
Value: "library/ubuntu:release-16.04",
|
||||||
|
},
|
||||||
|
models.FilterItem{
|
||||||
|
Kind: replication.FilterItemKindTag,
|
||||||
|
Value: "library/ubuntu:test",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
result := pfilter.DoFilter(items)
|
||||||
|
assert.Equal(t, 2, len(result))
|
||||||
|
assert.Equal(t, replication.FilterItemKindTag, result[0].Kind)
|
||||||
|
assert.Equal(t, "library/ubuntu:release-14.04", result[0].Value)
|
||||||
|
assert.Equal(t, replication.FilterItemKindTag, result[1].Kind)
|
||||||
|
assert.Equal(t, "library/ubuntu:release-16.04", result[1].Value)
|
||||||
|
|
||||||
|
}
|
75
src/replication/source/tag_combination_filter.go
Normal file
75
src/replication/source/tag_combination_filter.go
Normal file
@ -0,0 +1,75 @@
|
|||||||
|
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package source
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/vmware/harbor/src/common/utils/log"
|
||||||
|
"github.com/vmware/harbor/src/replication"
|
||||||
|
"github.com/vmware/harbor/src/replication/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TagCombinationFilter implements Filter interface for merging tag filter items
|
||||||
|
// whose repository are same into one repository filter item
|
||||||
|
type TagCombinationFilter struct{}
|
||||||
|
|
||||||
|
// NewTagCombinationFilter returns an instance of TagCombinationFilter
|
||||||
|
func NewTagCombinationFilter() *TagCombinationFilter {
|
||||||
|
return &TagCombinationFilter{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Init the filter. nil implement for now
|
||||||
|
func (t *TagCombinationFilter) Init() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetConvertor returns the convertor
|
||||||
|
func (t *TagCombinationFilter) GetConvertor() Convertor {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DoFilter filters resources
|
||||||
|
func (t *TagCombinationFilter) DoFilter(filterItems []models.FilterItem) []models.FilterItem {
|
||||||
|
repos := map[string][]string{}
|
||||||
|
for _, item := range filterItems {
|
||||||
|
if item.Kind != replication.FilterItemKindTag {
|
||||||
|
log.Warningf("unexpected filter item kind, expected: %s, got: %s, skip",
|
||||||
|
replication.FilterItemKindTag, item.Kind)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
strs := strings.Split(item.Value, ":")
|
||||||
|
if len(strs) != 2 {
|
||||||
|
log.Warningf("unexpected image format: %s, skip", item.Value)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
repos[strs[0]] = append(repos[strs[0]], strs[1])
|
||||||
|
}
|
||||||
|
|
||||||
|
items := []models.FilterItem{}
|
||||||
|
for repo, tags := range repos {
|
||||||
|
items = append(items, models.FilterItem{
|
||||||
|
Kind: replication.FilterItemKindRepository,
|
||||||
|
Value: repo,
|
||||||
|
Metadata: map[string]interface{}{
|
||||||
|
"tags": tags,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return items
|
||||||
|
}
|
83
src/replication/source/tag_combination_filter_test.go
Normal file
83
src/replication/source/tag_combination_filter_test.go
Normal file
@ -0,0 +1,83 @@
|
|||||||
|
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package source
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/vmware/harbor/src/replication"
|
||||||
|
"github.com/vmware/harbor/src/replication/models"
|
||||||
|
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
var tcfilter = NewTagCombinationFilter()
|
||||||
|
|
||||||
|
func TestTagCombinationFilteInit(t *testing.T) {
|
||||||
|
assert.Nil(t, tcfilter.Init())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTagCombinationFilteGetConvertor(t *testing.T) {
|
||||||
|
assert.Nil(t, tcfilter.GetConvertor())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTagCombinationFilteDoFilter(t *testing.T) {
|
||||||
|
items := []models.FilterItem{
|
||||||
|
models.FilterItem{
|
||||||
|
Kind: replication.FilterItemKindProject,
|
||||||
|
},
|
||||||
|
models.FilterItem{
|
||||||
|
Kind: replication.FilterItemKindRepository,
|
||||||
|
},
|
||||||
|
models.FilterItem{
|
||||||
|
Kind: replication.FilterItemKindTag,
|
||||||
|
Value: "library/ubuntu:invalid_tag:latest",
|
||||||
|
},
|
||||||
|
models.FilterItem{
|
||||||
|
Kind: replication.FilterItemKindTag,
|
||||||
|
Value: "library/ubuntu:14.04",
|
||||||
|
},
|
||||||
|
models.FilterItem{
|
||||||
|
Kind: replication.FilterItemKindTag,
|
||||||
|
Value: "library/ubuntu:16.04",
|
||||||
|
},
|
||||||
|
models.FilterItem{
|
||||||
|
Kind: replication.FilterItemKindTag,
|
||||||
|
Value: "library/centos:7",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
result := tcfilter.DoFilter(items)
|
||||||
|
assert.Equal(t, 2, len(result))
|
||||||
|
|
||||||
|
var ubuntu, centos models.FilterItem
|
||||||
|
if result[0].Value == "library/ubuntu" {
|
||||||
|
ubuntu = result[0]
|
||||||
|
centos = result[1]
|
||||||
|
} else {
|
||||||
|
centos = result[0]
|
||||||
|
ubuntu = result[1]
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, replication.FilterItemKindRepository, ubuntu.Kind)
|
||||||
|
assert.Equal(t, "library/ubuntu", ubuntu.Value)
|
||||||
|
metadata, ok := ubuntu.Metadata["tags"].([]string)
|
||||||
|
assert.True(t, ok)
|
||||||
|
assert.EqualValues(t, []string{"14.04", "16.04"}, metadata)
|
||||||
|
|
||||||
|
assert.Equal(t, replication.FilterItemKindRepository, centos.Kind)
|
||||||
|
assert.Equal(t, "library/centos", centos.Value)
|
||||||
|
metadata, ok = centos.Metadata["tags"].([]string)
|
||||||
|
assert.True(t, ok)
|
||||||
|
assert.EqualValues(t, []string{"7"}, metadata)
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user