Merge pull request #10961 from ywk253100/200306_replication

Make replication work with new artifact(phase 2)
This commit is contained in:
Wenkai Yin(尹文开) 2020-03-09 11:58:26 +08:00 committed by GitHub
commit c0542776e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 63 additions and 41 deletions

View File

@ -98,6 +98,7 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
info := &model.RegistryInfo{
Type: model.RegistryTypeHarbor,
SupportedResourceTypes: []model.ResourceType{
model.ResourceTypeArtifact,
model.ResourceTypeImage,
},
SupportedResourceFilters: []*model.FilterStyle{

View File

@ -45,9 +45,10 @@ func TestInfo(t *testing.T) {
assert.Equal(t, model.RegistryTypeHarbor, info.Type)
assert.Equal(t, 2, len(info.SupportedResourceFilters))
assert.Equal(t, 2, len(info.SupportedTriggers))
assert.Equal(t, 2, len(info.SupportedResourceTypes))
assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0])
assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[1])
assert.Equal(t, 3, len(info.SupportedResourceTypes))
assert.Equal(t, model.ResourceTypeArtifact, info.SupportedResourceTypes[0])
assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[1])
assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[2])
server.Close()
// chart museum disabled
@ -69,8 +70,9 @@ func TestInfo(t *testing.T) {
assert.Equal(t, model.RegistryTypeHarbor, info.Type)
assert.Equal(t, 2, len(info.SupportedResourceFilters))
assert.Equal(t, 2, len(info.SupportedTriggers))
assert.Equal(t, 1, len(info.SupportedResourceTypes))
assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0])
assert.Equal(t, 2, len(info.SupportedResourceTypes))
assert.Equal(t, model.ResourceTypeArtifact, info.SupportedResourceTypes[0])
assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[1])
server.Close()
}

View File

@ -17,6 +17,7 @@ package filter
import (
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
"strings"
)
// DoFilterArtifacts filter the artifacts according to the filters
@ -42,6 +43,13 @@ func BuildArtifactFilters(filters []*model.Filter) (ArtifactFilters, error) {
f = &artifactTagFilter{
pattern: filter.Value.(string),
}
case model.FilterTypeResource:
v := filter.Value.(model.ResourceType)
if v != model.ResourceTypeArtifact && v != model.ResourceTypeChart {
f = &artifactTypeFilter{
types: []string{string(v)},
}
}
}
if f != nil {
fs = append(fs, f)
@ -81,7 +89,7 @@ func (a *artifactTypeFilter) Filter(artifacts []*model.Artifact) ([]*model.Artif
var result []*model.Artifact
for _, artifact := range artifacts {
for _, t := range a.types {
if artifact.Type == t {
if strings.ToLower(artifact.Type) == strings.ToLower(t) {
result = append(result, artifact)
continue
}

View File

@ -94,7 +94,7 @@ func (p *Policy) Valid(v *validation.Validation) {
}
if filter.Type == FilterTypeResource {
rt := ResourceType(value)
if !(rt == ResourceTypeImage || rt == ResourceTypeChart) {
if !(rt == ResourceTypeArtifact || rt == ResourceTypeImage || rt == ResourceTypeChart) {
v.SetError("filters", fmt.Sprintf("invalid resource filter: %s", value))
break
}

View File

@ -16,6 +16,7 @@ package model
// the resource type
const (
ResourceTypeArtifact ResourceType = "artifact"
ResourceTypeImage ResourceType = "image"
ResourceTypeChart ResourceType = "chart"
)

View File

@ -60,14 +60,11 @@ func initialize(policy *model.Policy) (adp.Adapter, adp.Adapter, error) {
// fetch resources from the source registry
func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resource, error) {
var resTypes []model.ResourceType
var filters []*model.Filter
for _, filter := range policy.Filters {
if filter.Type != model.FilterTypeResource {
filters = append(filters, filter)
continue
}
if filter.Type == model.FilterTypeResource {
resTypes = append(resTypes, filter.Value.(model.ResourceType))
}
}
if len(resTypes) == 0 {
info, err := adapter.Info()
if err != nil {
@ -76,33 +73,42 @@ func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resourc
resTypes = append(resTypes, info.SupportedResourceTypes...)
}
resources := []*model.Resource{}
// convert the adapter to different interfaces according to its required resource types
for _, typ := range resTypes {
var res []*model.Resource
var err error
if typ == model.ResourceTypeImage {
// images
fetchArtifact := false
fetchChart := false
for _, resType := range resTypes {
if resType == model.ResourceTypeChart {
fetchChart = true
continue
}
fetchArtifact = true
}
var resources []*model.Resource
// artifacts
if fetchArtifact {
reg, ok := adapter.(adp.ImageRegistry)
if !ok {
return nil, fmt.Errorf("the adapter doesn't implement the ImageRegistry interface")
}
res, err = reg.FetchImages(filters)
} else if typ == model.ResourceTypeChart {
res, err := reg.FetchImages(policy.Filters)
if err != nil {
return nil, fmt.Errorf("failed to fetch artifacts: %v", err)
}
resources = append(resources, res...)
log.Debug("fetch artifacts completed")
}
// charts
if fetchChart {
reg, ok := adapter.(adp.ChartRegistry)
if !ok {
return nil, fmt.Errorf("the adapter doesn't implement the ChartRegistry interface")
}
res, err = reg.FetchCharts(filters)
} else {
return nil, fmt.Errorf("unsupported resource type %s", typ)
}
res, err := reg.FetchCharts(policy.Filters)
if err != nil {
return nil, fmt.Errorf("failed to fetch %s: %v", typ, err)
return nil, fmt.Errorf("failed to fetch charts: %v", err)
}
resources = append(resources, res...)
log.Debugf("fetch %s completed", typ)
log.Debug("fetch charts completed")
}
log.Debug("fetch resources from the source registry completed")
@ -290,7 +296,7 @@ func getResourceName(res *model.Resource) string {
n = len(meta.Vtags)
}
return fmt.Sprintf("%s [%d in total]", meta.Repository.Name, n)
return fmt.Sprintf("%s [%d artifact(s) in total]", meta.Repository.Name, n)
}
// repository:c namespace:n -> n/c

View File

@ -16,6 +16,8 @@ package image
import (
"errors"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"strings"
@ -214,9 +216,11 @@ func (t *transfer) copyImage(srcRepo, srcRef, dstRepo, dstRef string, override b
func (t *transfer) copyContent(content distribution.Descriptor, srcRepo, dstRepo string) error {
digest := content.Digest.String()
switch content.MediaType {
// when the media type of pulled manifest is manifest list,
// the contents it contains are a few manifests
case v1.MediaTypeImageManifest, schema2.MediaTypeManifest:
// when the media type of pulled manifest is index,
// the contents it contains are a few manifests/indexes
case v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList,
v1.MediaTypeImageManifest, schema2.MediaTypeManifest,
schema1.MediaTypeSignedManifest:
// as using digest as the reference, so set the override to true directly
return t.copyImage(srcRepo, digest, dstRepo, digest, true)
// handle foreign layer