Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-17 15:55:26 +01:00
Code Issues Projects Releases Wiki Activity

Replicate just one image if the media type is manifest list (#7602)

Browse Source 
Replicate just one image if the media type is manifest list

Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin 2019-05-07 10:49:10 +08:00 committed by Wang Yan
parent d27a6c0335
commit b0e3b65ade
1 changed files with 119 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

174
src/replication/transfer/image/transfer.go
View File

@ -16,8 +16,11 @@ package image
import (
"errors"
"fmt"
"strings"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
@ -136,7 +139,7 @@ func (t *transfer) copy(src *repository, dst *repository, override bool) error {
srcRepo, strings.Join(src.tags, ","), dstRepo, strings.Join(dst.tags, ","))
var err error
for i := range src.tags {
if e := t.copyTag(srcRepo, src.tags[i], dstRepo, dst.tags[i], override); e != nil {
if e := t.copyImage(srcRepo, src.tags[i], dstRepo, dst.tags[i], override); err != nil {
t.logger.Errorf(e.Error())
err = e
}
@ -150,17 +153,17 @@ func (t *transfer) copy(src *repository, dst *repository, override bool) error {
return nil
}
func (t *transfer) copyTag(srcRepo, srcTag, dstRepo, dstTag string, override bool) error {
func (t *transfer) copyImage(srcRepo, srcRef, dstRepo, dstRef string, override bool) error {
t.logger.Infof("copying %s:%s(source registry) to %s:%s(destination registry)...",
srcRepo, srcTag, dstRepo, dstTag)
srcRepo, srcRef, dstRepo, dstRef)
// pull the manifest from the source registry
manifest, digest, err := t.pullManifest(srcRepo, srcTag)
manifest, digest, err := t.pullManifest(srcRepo, srcRef)
if err != nil {
return err
}
// check the existence of the image on the destination registry
exist, digest2, err := t.exist(dstRepo, dstTag)
exist, digest2, err := t.exist(dstRepo, dstRef)
if err != nil {
return err
}
@ -168,51 +171,148 @@ func (t *transfer) copyTag(srcRepo, srcTag, dstRepo, dstTag string, override boo
// the same image already exists
if digest == digest2 {
t.logger.Infof("the image %s:%s already exists on the destination registry, skip",
dstRepo, dstTag)
dstRepo, dstRef)
return nil
}
// the same name image exists, but not allowed to override
if !override {
t.logger.Warningf("the same name image %s:%s exists on the destination registry, but the \"override\" is set to false, skip",
dstRepo, dstTag)
dstRepo, dstRef)
return nil
}
// the same name image exists, but allowed to override
t.logger.Warningf("the same name image %s:%s exists on the destination registry and the \"override\" is set to true, continue...",
dstRepo, dstTag)
dstRepo, dstRef)
}
// copy blobs between the source and destination registries
if err = t.copyBlobs(manifest.References(), srcRepo, dstRepo); err != nil {
return err
// copy contents between the source and destination registries
for _, content := range manifest.References() {
if err = t.copyContent(content, srcRepo, dstRepo); err != nil {
return err
}
}
// push the manifest to the destination registry
if err := t.pushManifest(manifest, dstRepo, dstTag); err != nil {
if err := t.pushManifest(manifest, dstRepo, dstRef); err != nil {
return err
}
t.logger.Infof("copy %s:%s(source registry) to %s:%s(destination registry) completed",
srcRepo, srcTag, dstRepo, dstTag)
srcRepo, srcRef, dstRepo, dstRef)
return nil
}
func (t *transfer) pullManifest(repository, tag string) (
// copy the content from source registry to destination according to its media type
func (t *transfer) copyContent(content distribution.Descriptor, srcRepo, dstRepo string) error {
digest := content.Digest.String()
switch content.MediaType {
// when the media type of pulled manifest is manifest list,
// the contents it contains are a few manifests
case schema2.MediaTypeManifest:
// as using digest as the reference, so set the override to true directly
return t.copyImage(srcRepo, digest, dstRepo, digest, true)
// copy layer or image config
case schema2.MediaTypeLayer, schema2.MediaTypeImageConfig:
return t.copyBlob(srcRepo, dstRepo, digest)
// handle foreign layer
case schema2.MediaTypeForeignLayer:
t.logger.Infof("the layer %s is a foreign layer, skip", digest)
return nil
// others
default:
err := fmt.Errorf("unsupported media type: %s", content.MediaType)
t.logger.Error(err.Error())
return err
}
}
// copy the layer or image config from the source registry to destination
func (t *transfer) copyBlob(srcRepo, dstRepo, digest string) error {
if t.shouldStop() {
return nil
}
t.logger.Infof("copying the blob %s...", digest)
exist, err := t.dst.BlobExist(dstRepo, digest)
if err != nil {
t.logger.Errorf("failed to check the existence of blob %s on the destination registry: %v", digest, err)
return err
}
if exist {
t.logger.Infof("the blob %s already exists on the destination registry, skip", digest)
return nil
}
size, data, err := t.src.PullBlob(srcRepo, digest)
if err != nil {
t.logger.Errorf("failed to pulling the blob %s: %v", digest, err)
return err
}
defer data.Close()
if err = t.dst.PushBlob(dstRepo, digest, size, data); err != nil {
t.logger.Errorf("failed to pushing the blob %s: %v", digest, err)
return err
}
t.logger.Infof("copy the blob %s completed", digest)
return nil
}
func (t *transfer) pullManifest(repository, reference string) (
distribution.Manifest, string, error) {
if t.shouldStop() {
return nil, "", nil
}
t.logger.Infof("pulling the manifest of image %s:%s ...", repository, tag)
manifest, digest, err := t.src.PullManifest(repository, tag, []string{
t.logger.Infof("pulling the manifest of image %s:%s ...", repository, reference)
manifest, digest, err := t.src.PullManifest(repository, reference, []string{
schema1.MediaTypeManifest,
schema2.MediaTypeManifest,
manifestlist.MediaTypeManifestList,
})
if err != nil {
t.logger.Errorf("failed to pull the manifest of image %s:%s: %v", repository, tag, err)
t.logger.Errorf("failed to pull the manifest of image %s:%s: %v", repository, reference, err)
return nil, "", err
}
t.logger.Infof("the manifest of image %s:%s pulled", repository, tag)
return manifest, digest, nil
t.logger.Infof("the manifest of image %s:%s pulled", repository, reference)
// this is a solution to work around that harbor doesn't support manifest list
return t.handleManifest(manifest, repository, digest)
}
// if the media type of the specified manifest is manifest list, just abstract one
// manifest from the list and return it
func (t *transfer) handleManifest(manifest distribution.Manifest, repository, digest string) (
distribution.Manifest, string, error) {
mediaType, _, err := manifest.Payload()
if err != nil {
t.logger.Errorf("failed to call the payload method for manifest of %s:%s: %v", repository, digest, err)
return nil, "", err
}
// manifest
if mediaType == schema1.MediaTypeManifest ||
mediaType == schema2.MediaTypeManifest {
return manifest, digest, nil
}
// manifest list
t.logger.Info("trying abstract a manifest from the manifest list...")
manifestlist, ok := manifest.(*manifestlist.DeserializedManifestList)
if !ok {
err := fmt.Errorf("the object isn't a DeserializedManifestList")
t.logger.Errorf(err.Error())
return nil, "", err
}
digest = ""
for _, reference := range manifestlist.Manifests {
if strings.ToLower(reference.Platform.Architecture) == "amd64" &&
strings.ToLower(reference.Platform.OS) == "linux" {
digest = reference.Digest.String()
t.logger.Infof("a manifest(architecture: amd64, os: linux) found, using this one: %s", digest)
break
}
}
if len(digest) == 0 {
digest = manifest.References()[0].Digest.String()
t.logger.Infof("no manifest(architecture: amd64, os: linux) found, using the first one: %s", digest)
}
return t.pullManifest(repository, digest)
}
func (t *transfer) exist(repository, tag string) (bool, string, error) {
@ -225,42 +325,6 @@ func (t *transfer) exist(repository, tag string) (bool, string, error) {
return exist, digest, nil
}
func (t *transfer) copyBlobs(blobs []distribution.Descriptor, srcRepo, dstRepo string) error {
for _, blob := range blobs {
if t.shouldStop() {
return nil
}
digest := blob.Digest.String()
if blob.MediaType == schema2.MediaTypeForeignLayer {
t.logger.Infof("the blob %s is a foreign layer, skip", digest)
continue
}
t.logger.Infof("copying the blob %s...", digest)
exist, err := t.dst.BlobExist(dstRepo, digest)
if err != nil {
t.logger.Errorf("failed to check the existence of blob %s on the destination registry: %v", digest, err)
return err
}
if exist {
t.logger.Infof("the blob %s already exists on the destination registry, skip", digest)
continue
}
size, data, err := t.src.PullBlob(srcRepo, digest)
if err != nil {
t.logger.Errorf("failed to pulling the blob %s: %v", digest, err)
return err
}
defer data.Close()
if err = t.dst.PushBlob(dstRepo, digest, size, data); err != nil {
t.logger.Errorf("failed to pushing the blob %s: %v", digest, err)
return err
}
t.logger.Infof("copy the blob %s completed", digest)
}
return nil
}
func (t *transfer) pushManifest(manifest distribution.Manifest, repository, tag string) error {
if t.shouldStop() {
return nil