Merge pull request #7357 from ywk253100/190412_harbor_adapter

Add filter support when fetching resources for Harbor adapter

src/replication/ng/adapter/harbor/chart_registry.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 
 	common_http "github.com/goharbor/harbor/src/common/http"
+	"github.com/goharbor/harbor/src/common/utils"
 	"github.com/goharbor/harbor/src/replication/ng/model"
 )
 
@@ -33,6 +34,21 @@ type chart struct {
 	Name string `json:"name"`
 }
 
+func (c *chart) Match(filters []*model.Filter) (bool, error) {
+	supportedFilters := []*model.Filter{}
+	for _, filter := range filters {
+		if filter.Type == model.FilterTypeName {
+			supportedFilters = append(supportedFilters, filter)
+		}
+	}
+	// trim the project part
+	_, name := utils.ParseRepository(c.Name)
+	item := &FilterItem{
+		Value: name,
+	}
+	return item.Match(supportedFilters)
+}
+
 type chartVersion struct {
 	Name    string `json:"name"`
 	Version string `json:"version"`
@@ -40,6 +56,19 @@ type chartVersion struct {
 	// Labels string `json:"labels"`
 }
 
+func (c *chartVersion) Match(filters []*model.Filter) (bool, error) {
+	supportedFilters := []*model.Filter{}
+	for _, filter := range filters {
+		if filter.Type == model.FilterTypeTag {
+			supportedFilters = append(supportedFilters, filter)
+		}
+	}
+	item := &FilterItem{
+		Value: c.Version,
+	}
+	return item.Match(supportedFilters)
+}
+
 type chartVersionDetail struct {
 	Metadata *chartVersionMetadata `json:"metadata"`
 }
@@ -56,12 +85,20 @@ func (a *adapter) FetchCharts(namespaces []string, filters []*model.Filter) ([]*
 		if err := a.client.Get(url, &charts); err != nil {
 			return nil, err
 		}
+		charts, err := filterCharts(charts, filters)
+		if err != nil {
+			return nil, err
+		}
 		for _, chart := range charts {
 			url := fmt.Sprintf("%s/api/chartrepo/%s/charts/%s", a.coreServiceURL, namespace, chart.Name)
 			chartVersions := []*chartVersion{}
 			if err := a.client.Get(url, &chartVersions); err != nil {
 				return nil, err
 			}
+			chartVersions, err = filterChartVersions(chartVersions, filters)
+			if err != nil {
+				return nil, err
+			}
 			for _, version := range chartVersions {
 				resources = append(resources, &model.Resource{
 					Type:     model.ResourceTypeChart,
@@ -195,6 +232,7 @@ func (a *adapter) DeleteChart(name, version string) error {
 	return a.client.Delete(url)
 }
 
+// TODO merge this method and utils.ParseRepository?
 func parseChartName(name string) (string, string, error) {
 	strs := strings.Split(name, "/")
 	if len(strs) == 2 && len(strs[0]) > 0 && len(strs[1]) > 0 {
@@ -202,3 +240,31 @@ func parseChartName(name string) (string, string, error) {
 	}
 	return "", "", fmt.Errorf("invalid chart name format: %s", name)
 }
+
+func filterCharts(charts []*chart, filters []*model.Filter) ([]*chart, error) {
+	result := []*chart{}
+	for _, chart := range charts {
+		match, err := chart.Match(filters)
+		if err != nil {
+			return nil, err
+		}
+		if match {
+			result = append(result, chart)
+		}
+	}
+	return result, nil
+}
+
+func filterChartVersions(chartVersions []*chartVersion, filters []*model.Filter) ([]*chartVersion, error) {
+	result := []*chartVersion{}
+	for _, chartVersion := range chartVersions {
+		match, err := chartVersion.Match(filters)
+		if err != nil {
+			return nil, err
+		}
+		if match {
+			result = append(result, chartVersion)
+		}
+	}
+	return result, nil
+}

src/replication/ng/adapter/harbor/filter.go

@@ -0,0 +1,59 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package harbor
+
+import (
+	"fmt"
+
+	"github.com/goharbor/harbor/src/replication/ng/model"
+	"github.com/goharbor/harbor/src/replication/ng/util"
+)
+
+// TODO unify the filter logic from different adapters into one?
+// and move the code into a separated common package
+
+// Filterable defines the interface that an object should implement
+// if the object can be filtered
+type Filterable interface {
+	Match([]*model.Filter) (bool, error)
+}
+
+// FilterItem is a filterable object that can be used to match string pattern
+type FilterItem struct {
+	Value string
+}
+
+// Match ...
+func (f *FilterItem) Match(filters []*model.Filter) (bool, error) {
+	if len(filters) == 0 {
+		return true, nil
+	}
+	matched := true
+	for _, filter := range filters {
+		pattern, ok := filter.Value.(string)
+		if !ok {
+			return false, fmt.Errorf("the type of filter value isn't string: %v", filter)
+		}
+		m, err := util.Match(pattern, f.Value)
+		if err != nil {
+			return false, err
+		}
+		if !m {
+			matched = false
+			break
+		}
+	}
+	return matched, nil
+}

src/replication/ng/adapter/harbor/filter_test.go

@@ -0,0 +1,87 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package harbor
+
+import (
+	"testing"
+
+	"github.com/goharbor/harbor/src/replication/ng/model"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMatch(t *testing.T) {
+	// nil filters
+	item := &FilterItem{}
+	match, err := item.Match(nil)
+	require.Nil(t, err)
+	assert.True(t, match)
+	// contains filter whose value isn't string
+	item = &FilterItem{}
+	filters := []*model.Filter{
+		{
+			Type:  "test",
+			Value: 1,
+		},
+	}
+	match, err = item.Match(filters)
+	require.NotNil(t, err)
+	// both filters match
+	item = &FilterItem{
+		Value: "b/c",
+	}
+	filters = []*model.Filter{
+		{
+			Value: "b/*",
+		},
+		{
+			Value: "*/c",
+		},
+	}
+	match, err = item.Match(filters)
+	require.Nil(t, err)
+	assert.True(t, match)
+	// one filter matches and the other one doesn't
+	item = &FilterItem{
+		Value: "b/c",
+	}
+	filters = []*model.Filter{
+		{
+			Value: "b/*",
+		},
+		{
+			Value: "d",
+		},
+	}
+	match, err = item.Match(filters)
+	require.Nil(t, err)
+	assert.False(t, match)
+	// both filters don't match
+	item = &FilterItem{
+		Value: "b/c",
+	}
+	filters = []*model.Filter{
+		{
+			Value: "f",
+		},
+		{
+			Value: "d",
+		},
+	}
+	match, err = item.Match(filters)
+	require.Nil(t, err)
+	assert.False(t, match)
+}

src/replication/ng/adapter/harbor/image_registry.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/goharbor/harbor/src/common/utils"
 	"github.com/goharbor/harbor/src/replication/ng/model"
 )
 
@@ -25,11 +26,38 @@ type repository struct {
 	Name string `json:"name"`
 }
 
+func (r *repository) Match(filters []*model.Filter) (bool, error) {
+	supportedFilters := []*model.Filter{}
+	for _, filter := range filters {
+		if filter.Type == model.FilterTypeName {
+			supportedFilters = append(supportedFilters, filter)
+		}
+	}
+	// trim the project part
+	_, name := utils.ParseRepository(r.Name)
+	item := &FilterItem{
+		Value: name,
+	}
+	return item.Match(supportedFilters)
+}
+
 type tag struct {
 	Name string `json:"name"`
 }
 
-// TODO implement filter
+func (t *tag) Match(filters []*model.Filter) (bool, error) {
+	supportedFilters := []*model.Filter{}
+	for _, filter := range filters {
+		if filter.Type == model.FilterTypeTag {
+			supportedFilters = append(supportedFilters, filter)
+		}
+	}
+	item := &FilterItem{
+		Value: t.Name,
+	}
+	return item.Match(supportedFilters)
+}
+
 func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) {
 	resources := []*model.Resource{}
 	for _, namespace := range namespaces {
@@ -42,13 +70,23 @@ func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]*
 		if err = a.client.Get(url, &repositories); err != nil {
 			return nil, err
 		}
-
+		repositories, err = filterRepositories(repositories, filters)
+		if err != nil {
+			return nil, err
+		}
 		for _, repository := range repositories {
 			url := fmt.Sprintf("%s/api/repositories/%s/tags", a.coreServiceURL, repository.Name)
 			tags := []*tag{}
 			if err = a.client.Get(url, &tags); err != nil {
 				return nil, err
 			}
+			tags, err = filterTags(tags, filters)
+			if err != nil {
+				return nil, err
+			}
+			if len(tags) == 0 {
+				continue
+			}
 			vtags := []string{}
 			for _, tag := range tags {
 				vtags = append(vtags, tag.Name)
@@ -79,3 +117,31 @@ func (a *adapter) DeleteManifest(repository, reference string) error {
 	url := fmt.Sprintf("%s/api/repositories/%s/tags/%s", a.coreServiceURL, repository, reference)
 	return a.client.Delete(url)
 }
+
+func filterRepositories(repositories []*repository, filters []*model.Filter) ([]*repository, error) {
+	result := []*repository{}
+	for _, repository := range repositories {
+		match, err := repository.Match(filters)
+		if err != nil {
+			return nil, err
+		}
+		if match {
+			result = append(result, repository)
+		}
+	}
+	return result, nil
+}
+
+func filterTags(tags []*tag, filters []*model.Filter) ([]*tag, error) {
+	result := []*tag{}
+	for _, tag := range tags {
+		match, err := tag.Match(filters)
+		if err != nil {
+			return nil, err
+		}
+		if match {
+			result = append(result, tag)
+		}
+	}
+	return result, nil
+}

src/replication/ng/transfer/repository/transfer.go

@@ -225,6 +225,10 @@ func (t *transfer) copyBlobs(blobs []distribution.Descriptor, srcRepo, dstRepo s
 			return jobStoppedErr
 		}
 		digest := blob.Digest.String()
+		if blob.MediaType == schema2.MediaTypeForeignLayer {
+			t.logger.Infof("the blob %s is a foreign layer, skip", digest)
+			continue
+		}
 		t.logger.Infof("copying the blob %s...", digest)
 		exist, err := t.dst.BlobExist(dstRepo, digest)
 		if err != nil {