Merge pull request #10961 from ywk253100/200306_replication

Make replication work with new artifact(phase 2)

src/replication/adapter/harbor/adapter.go

@@ -98,6 +98,7 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
 	info := &model.RegistryInfo{
 		Type: model.RegistryTypeHarbor,
 		SupportedResourceTypes: []model.ResourceType{
+			model.ResourceTypeArtifact,
 			model.ResourceTypeImage,
 		},
 		SupportedResourceFilters: []*model.FilterStyle{

src/replication/adapter/harbor/adapter_test.go

@@ -45,9 +45,10 @@ func TestInfo(t *testing.T) {
 	assert.Equal(t, model.RegistryTypeHarbor, info.Type)
 	assert.Equal(t, 2, len(info.SupportedResourceFilters))
 	assert.Equal(t, 2, len(info.SupportedTriggers))
-	assert.Equal(t, 2, len(info.SupportedResourceTypes))
-	assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0])
-	assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[1])
+	assert.Equal(t, 3, len(info.SupportedResourceTypes))
+	assert.Equal(t, model.ResourceTypeArtifact, info.SupportedResourceTypes[0])
+	assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[1])
+	assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[2])
 	server.Close()
 
 	// chart museum disabled
@@ -69,8 +70,9 @@ func TestInfo(t *testing.T) {
 	assert.Equal(t, model.RegistryTypeHarbor, info.Type)
 	assert.Equal(t, 2, len(info.SupportedResourceFilters))
 	assert.Equal(t, 2, len(info.SupportedTriggers))
-	assert.Equal(t, 1, len(info.SupportedResourceTypes))
-	assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0])
+	assert.Equal(t, 2, len(info.SupportedResourceTypes))
+	assert.Equal(t, model.ResourceTypeArtifact, info.SupportedResourceTypes[0])
+	assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[1])
 	server.Close()
 }
 

src/replication/filter/artifact.go

@@ -17,6 +17,7 @@ package filter
 import (
 	"github.com/goharbor/harbor/src/replication/model"
 	"github.com/goharbor/harbor/src/replication/util"
+	"strings"
 )
 
 // DoFilterArtifacts filter the artifacts according to the filters
@@ -42,6 +43,13 @@ func BuildArtifactFilters(filters []*model.Filter) (ArtifactFilters, error) {
 			f = &artifactTagFilter{
 				pattern: filter.Value.(string),
 			}
+		case model.FilterTypeResource:
+			v := filter.Value.(model.ResourceType)
+			if v != model.ResourceTypeArtifact && v != model.ResourceTypeChart {
+				f = &artifactTypeFilter{
+					types: []string{string(v)},
+				}
+			}
 		}
 		if f != nil {
 			fs = append(fs, f)
@@ -81,7 +89,7 @@ func (a *artifactTypeFilter) Filter(artifacts []*model.Artifact) ([]*model.Artif
 	var result []*model.Artifact
 	for _, artifact := range artifacts {
 		for _, t := range a.types {
-			if artifact.Type == t {
+			if strings.ToLower(artifact.Type) == strings.ToLower(t) {
 				result = append(result, artifact)
 				continue
 			}

src/replication/model/policy.go

@@ -94,7 +94,7 @@ func (p *Policy) Valid(v *validation.Validation) {
 			}
 			if filter.Type == FilterTypeResource {
 				rt := ResourceType(value)
-				if !(rt == ResourceTypeImage || rt == ResourceTypeChart) {
+				if !(rt == ResourceTypeArtifact || rt == ResourceTypeImage || rt == ResourceTypeChart) {
 					v.SetError("filters", fmt.Sprintf("invalid resource filter: %s", value))
 					break
 				}

src/replication/model/resource.go

@@ -16,8 +16,9 @@ package model
 
 // the resource type
 const (
-	ResourceTypeImage ResourceType = "image"
-	ResourceTypeChart ResourceType = "chart"
+	ResourceTypeArtifact ResourceType = "artifact"
+	ResourceTypeImage    ResourceType = "image"
+	ResourceTypeChart    ResourceType = "chart"
 )
 
 // ResourceType represents the type of the resource

src/replication/operation/flow/stage.go

@@ -60,13 +60,10 @@ func initialize(policy *model.Policy) (adp.Adapter, adp.Adapter, error) {
 // fetch resources from the source registry
 func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resource, error) {
 	var resTypes []model.ResourceType
-	var filters []*model.Filter
 	for _, filter := range policy.Filters {
-		if filter.Type != model.FilterTypeResource {
-			filters = append(filters, filter)
-			continue
+		if filter.Type == model.FilterTypeResource {
+			resTypes = append(resTypes, filter.Value.(model.ResourceType))
 		}
-		resTypes = append(resTypes, filter.Value.(model.ResourceType))
 	}
 	if len(resTypes) == 0 {
 		info, err := adapter.Info()
@@ -76,33 +73,42 @@ func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resourc
 		resTypes = append(resTypes, info.SupportedResourceTypes...)
 	}
 
-	resources := []*model.Resource{}
-	// convert the adapter to different interfaces according to its required resource types
-	for _, typ := range resTypes {
-		var res []*model.Resource
-		var err error
-		if typ == model.ResourceTypeImage {
-			// images
-			reg, ok := adapter.(adp.ImageRegistry)
-			if !ok {
-				return nil, fmt.Errorf("the adapter doesn't implement the ImageRegistry interface")
-			}
-			res, err = reg.FetchImages(filters)
-		} else if typ == model.ResourceTypeChart {
-			// charts
-			reg, ok := adapter.(adp.ChartRegistry)
-			if !ok {
-				return nil, fmt.Errorf("the adapter doesn't implement the ChartRegistry interface")
-			}
-			res, err = reg.FetchCharts(filters)
-		} else {
-			return nil, fmt.Errorf("unsupported resource type %s", typ)
+	fetchArtifact := false
+	fetchChart := false
+	for _, resType := range resTypes {
+		if resType == model.ResourceTypeChart {
+			fetchChart = true
+			continue
 		}
+		fetchArtifact = true
+	}
+
+	var resources []*model.Resource
+	// artifacts
+	if fetchArtifact {
+		reg, ok := adapter.(adp.ImageRegistry)
+		if !ok {
+			return nil, fmt.Errorf("the adapter doesn't implement the ImageRegistry interface")
+		}
+		res, err := reg.FetchImages(policy.Filters)
 		if err != nil {
-			return nil, fmt.Errorf("failed to fetch %s: %v", typ, err)
+			return nil, fmt.Errorf("failed to fetch artifacts: %v", err)
 		}
 		resources = append(resources, res...)
-		log.Debugf("fetch %s completed", typ)
+		log.Debug("fetch artifacts completed")
+	}
+	// charts
+	if fetchChart {
+		reg, ok := adapter.(adp.ChartRegistry)
+		if !ok {
+			return nil, fmt.Errorf("the adapter doesn't implement the ChartRegistry interface")
+		}
+		res, err := reg.FetchCharts(policy.Filters)
+		if err != nil {
+			return nil, fmt.Errorf("failed to fetch charts: %v", err)
+		}
+		resources = append(resources, res...)
+		log.Debug("fetch charts completed")
 	}
 
 	log.Debug("fetch resources from the source registry completed")
@@ -290,7 +296,7 @@ func getResourceName(res *model.Resource) string {
 		n = len(meta.Vtags)
 	}
 
-	return fmt.Sprintf("%s [%d in total]", meta.Repository.Name, n)
+	return fmt.Sprintf("%s [%d artifact(s) in total]", meta.Repository.Name, n)
 }
 
 // repository:c namespace:n -> n/c

src/replication/transfer/image/transfer.go

@@ -16,6 +16,8 @@ package image
 
 import (
 	"errors"
+	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/schema1"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"strings"
 
@@ -214,9 +216,11 @@ func (t *transfer) copyImage(srcRepo, srcRef, dstRepo, dstRef string, override b
 func (t *transfer) copyContent(content distribution.Descriptor, srcRepo, dstRepo string) error {
 	digest := content.Digest.String()
 	switch content.MediaType {
-	// when the media type of pulled manifest is manifest list,
-	// the contents it contains are a few manifests
-	case v1.MediaTypeImageManifest, schema2.MediaTypeManifest:
+	// when the media type of pulled manifest is index,
+	// the contents it contains are a few manifests/indexes
+	case v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList,
+		v1.MediaTypeImageManifest, schema2.MediaTypeManifest,
+		schema1.MediaTypeSignedManifest:
 		// as using digest as the reference, so set the override to true directly
 		return t.copyImage(srcRepo, digest, dstRepo, digest, true)
 	// handle foreign layer