Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-14 22:35:36 +01:00
Code Issues Projects Releases Wiki Activity

Add content type and length in header

Browse Source 
Fixes #13740
Update ManifestExist to return Descriptor instead of digest
For docker 20.10 or containerd, it HEAD the manifest before pull, then
it GET the manifest with digest, add logic to handle this scenario and
correlate the tag between the digest in proxy cache

Signed-off-by: stonezdj <stonezdj@gmail.com>
This commit is contained in:
stonezdj 2020-12-11 18:14:15 +08:00
parent 98f3c5d452
commit aa3002e7a5
16 changed files with 258 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/controller/proxy/controller.go
View File

@ -17,6 +17,7 @@ package proxy
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/controller/tag"
"io"
"strings"
"sync"
@ -43,7 +44,7 @@ const (
maxManifestWait = 10
sleepIntervalSec = 20
// keep manifest list in cache for one week
manifestListCacheIntervalSec = 7 * 24 * 60 * 60
manifestListCacheInterval = 7 * 24 * 60 * 60 * time.Second
)
var (
@ -65,7 +66,9 @@ type Controller interface {
// art is the ArtifactInfo which includes the tag or digest of the manifest
ProxyManifest(ctx context.Context, art lib.ArtifactInfo, remote RemoteInterface) (distribution.Manifest, error)
// HeadManifest send manifest head request to the remote server
HeadManifest(ctx context.Context, art lib.ArtifactInfo, remote RemoteInterface) (bool, string, error)
HeadManifest(ctx context.Context, art lib.ArtifactInfo, remote RemoteInterface) (bool, *distribution.Descriptor, error)
// EnsureTag ensure tag for digest
EnsureTag(ctx context.Context, art lib.ArtifactInfo, tagName string) error
}
type controller struct {
blobCtl blob.Controller
@ -90,6 +93,24 @@ func ControllerInstance() Controller {
return ctl
}
func (c *controller) EnsureTag(ctx context.Context, art lib.ArtifactInfo, tagName string) error {
// search the digest in cache and query with trimmed digest
var trimmedDigest string
err := c.cache.Fetch(TrimmedManifestlist+art.Digest, &trimmedDigest)
if err != cache.ErrNotFound {
log.Debugf("Found trimed digest: %v", trimmedDigest)
art.Digest = trimmedDigest
}
a, err := c.local.GetManifest(ctx, art)
if err != nil {
return err
}
if a == nil {
return fmt.Errorf("the artifact is not ready yet, failed to tag it to %v", tagName)
}
return tag.Ctl.Ensure(ctx, a.RepositoryID, a.Artifact.ID, tagName)
}
func (c *controller) UseLocalBlob(ctx context.Context, art lib.ArtifactInfo) bool {
if len(art.Digest) == 0 {
return false
@ -119,11 +140,11 @@ func (c *controller) UseLocalManifest(ctx context.Context, art lib.ArtifactInfo,
}
remoteRepo := getRemoteRepo(art)
exist, dig, err := remote.ManifestExist(remoteRepo, getReference(art)) // HEAD
exist, desc, err := remote.ManifestExist(remoteRepo, getReference(art)) // HEAD
if err != nil {
return false, nil, err
}
if !exist {
if !exist || desc == nil {
go func() {
c.local.DeleteManifest(remoteRepo, art.Tag)
}()
@ -132,18 +153,18 @@ func (c *controller) UseLocalManifest(ctx context.Context, art lib.ArtifactInfo,
var content []byte
if c.cache != nil {
err = c.cache.Fetch(getManifestListKey(art.Repository, dig), &content)
err = c.cache.Fetch(getManifestListKey(art.Repository, string(desc.Digest)), &content)
if err == nil {
log.Debugf("Get the manifest list with key=cache:%v", getManifestListKey(art.Repository, dig))
return true, &ManifestList{content, dig, manifestlist.MediaTypeManifestList}, nil
log.Debugf("Get the manifest list with key=cache:%v", getManifestListKey(art.Repository, string(desc.Digest)))
return true, &ManifestList{content, string(desc.Digest), manifestlist.MediaTypeManifestList}, nil
}
if err == cache.ErrNotFound {
log.Debugf("Digest is not found in manifest list cache, key=cache:%v", getManifestListKey(art.Repository, dig))
log.Debugf("Digest is not found in manifest list cache, key=cache:%v", getManifestListKey(art.Repository, string(desc.Digest)))
} else {
log.Errorf("Failed to get manifest list from cache, error: %v", err)
}
}
return a != nil && dig == a.Digest, nil, nil // digest matches
return a != nil && string(desc.Digest) == a.Digest, nil, nil // digest matches
}
func getManifestListKey(repo, dig string) string {
@ -198,7 +219,7 @@ func (c *controller) ProxyManifest(ctx context.Context, art lib.ArtifactInfo, re
return man, nil
}
func (c *controller) HeadManifest(ctx context.Context, art lib.ArtifactInfo, remote RemoteInterface) (bool, string, error) {
func (c *controller) HeadManifest(ctx context.Context, art lib.ArtifactInfo, remote RemoteInterface) (bool, *distribution.Descriptor, error) {
remoteRepo := getRemoteRepo(art)
ref := getReference(art)
return remote.ManifestExist(remoteRepo, ref)
@ -247,7 +268,7 @@ func (c *controller) waitAndPushManifest(ctx context.Context, remoteRepo string,
}
key := getManifestListKey(art.Repository, art.Digest)
log.Debugf("Cache manifest list with key=cache:%v", key)
err = c.cache.Save(key, payload, manifestListCacheIntervalSec)
err = c.cache.Save(key, payload, manifestListCacheInterval)
if err != nil {
log.Errorf("failed to cache payload, error %v", err)
}

29
src/controller/proxy/controller_test.go
View File

@ -23,29 +23,14 @@ import (
"github.com/goharbor/harbor/src/lib"
_ "github.com/goharbor/harbor/src/lib/cache"
"github.com/goharbor/harbor/src/lib/errors"
testproxy "github.com/goharbor/harbor/src/testing/controller/proxy"
"github.com/opencontainers/go-digest"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"io"
"testing"
)
type remoteInterfaceMock struct {
mock.Mock
}
func (r *remoteInterfaceMock) BlobReader(repo, dig string) (int64, io.ReadCloser, error) {
panic("implement me")
}
func (r *remoteInterfaceMock) Manifest(repo string, ref string) (distribution.Manifest, string, error) {
panic("implement me")
}
func (r *remoteInterfaceMock) ManifestExist(repo, ref string) (bool, string, error) {
args := r.Called(repo, ref)
return args.Bool(0), args.String(1), args.Error(2)
}
type localInterfaceMock struct {
mock.Mock
}
@ -95,14 +80,14 @@ func (l *localInterfaceMock) DeleteManifest(repo, ref string) {
type proxyControllerTestSuite struct {
suite.Suite
local *localInterfaceMock
remote *remoteInterfaceMock
remote *testproxy.RemoteInterface
ctr Controller
proj *models.Project
}
func (p *proxyControllerTestSuite) SetupTest() {
p.local = &localInterfaceMock{}
p.remote = &remoteInterfaceMock{}
p.remote = &testproxy.RemoteInterface{}
p.proj = &models.Project{RegistryID: 1}
p.ctr = &controller{
blobCtl: blob.Ctl,
@ -125,8 +110,9 @@ func (p *proxyControllerTestSuite) TestUseLocalManifest_True() {
func (p *proxyControllerTestSuite) TestUseLocalManifest_False() {
ctx := context.Background()
dig := "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b"
desc := &distribution.Descriptor{Digest: digest.Digest(dig)}
art := lib.ArtifactInfo{Repository: "library/hello-world", Digest: dig}
p.remote.On("ManifestExist", mock.Anything, mock.Anything).Return(true, dig, nil)
p.remote.On("ManifestExist", mock.Anything, mock.Anything).Return(true, desc, nil)
p.local.On("GetManifest", mock.Anything, mock.Anything).Return(nil, nil)
result, _, err := p.ctr.UseLocalManifest(ctx, art, p.remote)
p.Assert().Nil(err)
@ -136,8 +122,9 @@ func (p *proxyControllerTestSuite) TestUseLocalManifest_False() {
func (p *proxyControllerTestSuite) TestUseLocalManifestWithTag_False() {
ctx := context.Background()
art := lib.ArtifactInfo{Repository: "library/hello-world", Tag: "latest"}
desc := &distribution.Descriptor{}
p.local.On("GetManifest", mock.Anything, mock.Anything).Return(&artifact.Artifact{}, nil)
p.remote.On("ManifestExist", mock.Anything, mock.Anything).Return(false, "", nil)
p.remote.On("ManifestExist", mock.Anything, mock.Anything).Return(false, desc, nil)
result, _, err := p.ctr.UseLocalManifest(ctx, art, p.remote)
p.Assert().True(errors.IsNotFoundErr(err))
p.Assert().False(result)

26
src/controller/proxy/local.go
View File

@ -17,7 +17,9 @@ package proxy
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/lib/cache"
"io"
"strings"
"time"
"github.com/docker/distribution"
@ -34,6 +36,9 @@ import (
"github.com/opencontainers/go-digest"
)
// TrimmedManifestlist - key prefix for trimmed manifest
const TrimmedManifestlist = "trimmedmanifestlist:"
// localInterface defines operations related to local repo under proxy mode
type localInterface interface {
// BlobExist check if the blob exist in local repo
@ -68,6 +73,7 @@ func (l *localHelper) GetManifest(ctx context.Context, art lib.ArtifactInfo) (*a
type localHelper struct {
registry registry.Client
artifactCtl artifactController
cache cache.Cache
}
type artifactController interface {
@ -93,6 +99,7 @@ func (l *localHelper) init() {
// the traffic is internal only
registryURL := config.LocalCoreURL()
l.registry = registry.NewClientWithAuthorizer(registryURL, secret.NewAuthorizer(), true)
l.cache = cache.Default()
}
func (l *localHelper) PushBlob(localRepo string, desc distribution.Descriptor, bReader io.ReadCloser) error {
@ -180,19 +187,34 @@ func (l *localHelper) PushManifestList(ctx context.Context, repo string, tag str
return err
}
log.Debugf("The manifest list payload: %v", string(pl))
dig := digest.FromBytes(pl)
newDig := digest.FromBytes(pl)
l.cacheTrimmedDigest(ctx, string(newDig))
// Because the manifest list maybe updated, need to recheck if it is exist in local
art := lib.ArtifactInfo{Repository: repo, Tag: tag}
a, err := l.GetManifest(ctx, art)
if err != nil {
return err
}
if a != nil && a.Digest == string(dig) {
if a != nil && a.Digest == string(newDig) {
return nil
}
// because current digest is changed, need to push to the new digest
if strings.HasPrefix(tag, "sha256:") {
tag = string(newDig)
}
return l.PushManifest(repo, tag, newMan)
}
func (l *localHelper) cacheTrimmedDigest(ctx context.Context, newDig string) {
if l.cache == nil {
return
}
art := lib.GetArtifactInfo(ctx)
key := TrimmedManifestlist + string(art.Digest)
log.Debugf("Saved key:%v, value:%v", key, newDig)
l.cache.Save(key, newDig)
}
func (l *localHelper) CheckDependencies(ctx context.Context, repo string, man distribution.Manifest) []distribution.Descriptor {
descriptors := man.References()
waitDesc := make([]distribution.Descriptor, 0)

4
src/controller/proxy/remote.go
View File

@ -30,7 +30,7 @@ type RemoteInterface interface {
// Manifest get manifest by reference
Manifest(repo string, ref string) (distribution.Manifest, string, error)
// ManifestExist checks manifest exist, if exist, return digest
ManifestExist(repo string, ref string) (bool, string, error)
ManifestExist(repo string, ref string) (bool, *distribution.Descriptor, error)
}
// remoteHelper defines operations related to remote repository under proxy
@ -86,6 +86,6 @@ func (r *remoteHelper) Manifest(repo string, ref string) (distribution.Manifest,
return r.registry.PullManifest(repo, ref)
}
func (r *remoteHelper) ManifestExist(repo string, ref string) (bool, string, error) {
func (r *remoteHelper) ManifestExist(repo string, ref string) (bool, *distribution.Descriptor, error) {
return r.registry.ManifestExist(repo, ref)
}

10
src/controller/replication/flow/mock_adapter_test.go
View File

@ -134,7 +134,7 @@ func (_m *mockAdapter) Info() (*model.RegistryInfo, error) {
}
// ManifestExist provides a mock function with given fields: repository, reference
func (_m *mockAdapter) ManifestExist(repository string, reference string) (bool, string, error) {
func (_m *mockAdapter) ManifestExist(repository string, reference string) (bool, *distribution.Descriptor, error) {
ret := _m.Called(repository, reference)
var r0 bool
@ -144,11 +144,13 @@ func (_m *mockAdapter) ManifestExist(repository string, reference string) (bool,
r0 = ret.Get(0).(bool)
}
var r1 string
if rf, ok := ret.Get(1).(func(string, string) string); ok {
var r1 *distribution.Descriptor
if rf, ok := ret.Get(1).(func(string, string) *distribution.Descriptor); ok {
r1 = rf(repository, reference)
} else {
r1 = ret.Get(1).(string)
if ret.Get(1) != nil {
r1 = ret.Get(1).(*distribution.Descriptor)
}
}
var r2 error

24
src/pkg/registry/client.go
View File

@ -72,7 +72,7 @@ type Client interface {
// ListTags lists the tags under the specified repository
ListTags(repository string) (tags []string, err error)
// ManifestExist checks the existence of the manifest
ManifestExist(repository, reference string) (exist bool, digest string, err error)
ManifestExist(repository, reference string) (exist bool, desc *distribution.Descriptor, err error)
// PullManifest pulls the specified manifest
PullManifest(repository, reference string, acceptedMediaTypes ...string) (manifest distribution.Manifest, digest string, err error)
// PushManifest pushes the specified manifest
@ -242,10 +242,10 @@ func (c *client) listTags(url string) ([]string, string, error) {
return tgs.Tags, next(resp.Header.Get("Link")), nil
}
func (c *client) ManifestExist(repository, reference string) (bool, string, error) {
func (c *client) ManifestExist(repository, reference string) (bool, *distribution.Descriptor, error) {
req, err := http.NewRequest(http.MethodHead, buildManifestURL(c.url, repository, reference), nil)
if err != nil {
return false, "", err
return false, nil, err
}
for _, mediaType := range accepts {
req.Header.Add(http.CanonicalHeaderKey("Accept"), mediaType)
@ -253,12 +253,16 @@ func (c *client) ManifestExist(repository, reference string) (bool, string, erro
resp, err := c.do(req)
if err != nil {
if errors.IsErr(err, errors.NotFoundCode) {
return false, "", nil
return false, nil, nil
}
return false, "", err
return false, nil, err
}
defer resp.Body.Close()
return true, resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")), nil
dig := resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
contentType := resp.Header.Get(http.CanonicalHeaderKey("Content-Type"))
contentLen := resp.Header.Get(http.CanonicalHeaderKey("Content-Length"))
len, _ := strconv.Atoi(contentLen)
return true, &distribution.Descriptor{Digest: digest.Digest(dig), MediaType: contentType, Size: int64(len)}, nil
}
func (c *client) PullManifest(repository, reference string, acceptedMediaTypes ...string) (
@ -310,7 +314,7 @@ func (c *client) DeleteManifest(repository, reference string) error {
_, err := digest.Parse(reference)
if err != nil {
// the reference is tag, get the digest first
exist, digest, err := c.ManifestExist(repository, reference)
exist, desc, err := c.ManifestExist(repository, reference)
if err != nil {
return err
}
@ -318,7 +322,7 @@ func (c *client) DeleteManifest(repository, reference string) error {
return errors.New(nil).WithCode(errors.NotFoundCode).
WithMessage("%s:%s not found", repository, reference)
}
reference = digest
reference = string(desc.Digest)
}
req, err := http.NewRequest(http.MethodDelete, buildManifestURL(c.url, repository, reference), nil)
if err != nil {
@ -450,13 +454,13 @@ func (c *client) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) er
}
// check the existence of the artifact on the destination repository
exist, dstDgt, err := c.ManifestExist(dstRepo, dstRef)
exist, desc, err := c.ManifestExist(dstRepo, dstRef)
if err != nil {
return err
}
if exist {
// the same artifact already exists
if srcDgt == dstDgt {
if desc != nil && srcDgt == string(desc.Digest) {
return nil
}
// the same name artifact exists, but not allowed to override

6
src/pkg/registry/client_test.go
View File

@ -171,15 +171,15 @@ func (c *clientTestSuite) TestManifestExist() {
client := NewClient(server.URL, "", "", true)
// doesn't exist
exist, digest, err := client.ManifestExist("library/alpine", "latest")
exist, desc, err := client.ManifestExist("library/alpine", "latest")
c.Require().Nil(err)
c.False(exist)
// exist
exist, digest, err = client.ManifestExist("library/hello-world", "latest")
exist, desc, err = client.ManifestExist("library/hello-world", "latest")
c.Require().Nil(err)
c.True(exist)
c.Equal("digest", digest)
c.Equal("digest", string(desc.Digest))
}
func (c *clientTestSuite) TestPullManifest() {

2
src/replication/adapter/adapter.go
View File

@ -54,7 +54,7 @@ type Adapter interface {
// ArtifactRegistry defines the capabilities that an artifact registry should have
type ArtifactRegistry interface {
FetchArtifacts(filters []*model.Filter) ([]*model.Resource, error)
ManifestExist(repository, reference string) (exist bool, digest string, err error)
ManifestExist(repository, reference string) (exist bool, desc *distribution.Descriptor, err error)
PullManifest(repository, reference string, accepttedMediaTypes ...string) (manifest distribution.Manifest, digest string, err error)
PushManifest(repository, reference, mediaType string, payload []byte) (string, error)
DeleteManifest(repository, reference string) error // the "reference" can be "tag" or "digest", the function needs to handle both

27
src/replication/adapter/huawei/image_registry.go
View File

@ -3,11 +3,12 @@ package huawei
import (
"encoding/json"
"fmt"
"github.com/docker/distribution"
"github.com/goharbor/harbor/src/replication/model"
"io/ioutil"
"net/http"
"strconv"
"time"
"github.com/goharbor/harbor/src/replication/model"
)
// FetchArtifacts gets resources from Huawei SWR
@ -54,17 +55,17 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
}
// ManifestExist check the manifest of Huawei SWR
func (a *adapter) ManifestExist(repository, reference string) (exist bool, digest string, err error) {
func (a *adapter) ManifestExist(repository, reference string) (exist bool, desc *distribution.Descriptor, err error) {
token, err := getJwtToken(a, repository)
if err != nil {
return exist, digest, err
return exist, nil, err
}
urls := fmt.Sprintf("%s/v2/%s/manifests/%s", a.registry.URL, repository, reference)
r, err := http.NewRequest("GET", urls, nil)
if err != nil {
return exist, digest, err
return exist, nil, err
}
r.Header.Add("content-type", "application/json; charset=utf-8")
@ -72,29 +73,33 @@ func (a *adapter) ManifestExist(repository, reference string) (exist bool, diges
resp, err := a.oriClient.Do(r)
if err != nil {
return exist, digest, err
return exist, nil, err
}
defer resp.Body.Close()
code := resp.StatusCode
if code >= 300 || code < 200 {
if code == 404 {
return false, digest, nil
return false, nil, nil
}
body, _ := ioutil.ReadAll(resp.Body)
return exist, digest, fmt.Errorf("[%d][%s]", code, string(body))
return exist, nil, fmt.Errorf("[%d][%s]", code, string(body))
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return exist, digest, err
return exist, nil, err
}
exist = true
manifest := hwManifest{}
err = json.Unmarshal(body, &manifest)
if err != nil {
return exist, digest, err
return exist, nil, err
}
return exist, manifest.Config.Digest, nil
contentType := resp.Header.Get(http.CanonicalHeaderKey("Content-Type"))
contentLen := resp.Header.Get(http.CanonicalHeaderKey("Content-Length"))
len, _ := strconv.Atoi(contentLen)
return exist, &distribution.Descriptor{MediaType: contentType, Size: int64(len)}, nil
}
// DeleteManifest delete the manifest of Huawei SWR

4
src/replication/adapter/huawei/image_registry_test.go
View File

@ -44,7 +44,7 @@ func TestAdapter_FetchArtifacts(t *testing.T) {
}
func TestAdapter_ManifestExist(t *testing.T) {
exist, digest, err := HWAdapter.ManifestExist("", "")
exist, desc, err := HWAdapter.ManifestExist("", "")
if err != nil {
if strings.HasPrefix(err.Error(), "[401]") {
t.Log("huawei ak/sk is not available", err.Error())
@ -53,7 +53,7 @@ func TestAdapter_ManifestExist(t *testing.T) {
}
} else {
if exist {
t.Log(digest)
t.Log(desc.Digest)
}
}
}

8
src/replication/transfer/image/transfer.go
View File

@ -328,13 +328,17 @@ func (t *transfer) pullManifest(repository, reference string) (
}
func (t *transfer) exist(repository, tag string) (bool, string, error) {
exist, digest, err := t.dst.ManifestExist(repository, tag)
exist, desc, err := t.dst.ManifestExist(repository, tag)
if err != nil {
t.logger.Errorf("failed to check the existence of the manifest of artifact %s:%s on the destination registry: %v",
repository, tag, err)
return false, "", err
}
return exist, digest, nil
var dig string
if desc != nil {
dig = string(desc.Digest)
}
return exist, dig, nil
}
func (t *transfer) pushManifest(manifest distribution.Manifest, repository, tag string) error {

7
src/replication/transfer/image/transfer_test.go
View File

@ -16,6 +16,7 @@ package image
import (
"bytes"
"github.com/opencontainers/go-digest"
"io"
"io/ioutil"
"testing"
@ -35,11 +36,11 @@ func (f *fakeRegistry) FetchArtifacts([]*model.Filter) ([]*model.Resource, error
return nil, nil
}
func (f *fakeRegistry) ManifestExist(repository, reference string) (bool, string, error) {
func (f *fakeRegistry) ManifestExist(repository, reference string) (bool, *distribution.Descriptor, error) {
if repository == "destination" && reference == "b1" {
return true, "sha256:c6b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", nil
return true, &distribution.Descriptor{Digest: digest.Digest("sha256:c6b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7")}, nil
}
return false, "sha256:c6b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", nil
return false, &distribution.Descriptor{Digest: digest.Digest("sha256:c6b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7")}, nil
}
func (f *fakeRegistry) PullManifest(repository, reference string, accepttedMediaTypes ...string) (distribution.Manifest, string, error) {
manifest := `{

34
src/server/middleware/repoproxy/proxy.go
View File

@ -17,9 +17,6 @@ package repoproxy
import (
"context"
"fmt"
"io"
"net/http"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/security"
"github.com/goharbor/harbor/src/common/security/proxycachesecret"
@ -29,9 +26,13 @@ import (
"github.com/goharbor/harbor/src/lib/errors"
httpLib "github.com/goharbor/harbor/src/lib/http"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/registry"
"github.com/goharbor/harbor/src/server/middleware"
"io"
"net/http"
"time"
)
var registryMgr = registry.NewDefaultManager()
@ -41,6 +42,8 @@ const (
contentType = "Content-Type"
dockerContentDigest = "Docker-Content-Digest"
etag = "Etag"
ensureTagInterval = 10 * time.Second
ensureTagMaxRetry = 60
)
// BlobGetMiddleware handle get blob request
@ -227,14 +230,31 @@ func DisableBlobAndManifestUploadMiddleware() func(http.Handler) http.Handler {
}
func proxyManifestHead(ctx context.Context, w http.ResponseWriter, ctl proxy.Controller, p *models.Project, art lib.ArtifactInfo, remote proxy.RemoteInterface) error {
exist, dig, err := ctl.HeadManifest(ctx, art, remote)
exist, desc, err := ctl.HeadManifest(ctx, art, remote)
if err != nil {
return err
}
if !exist {
if !exist || desc == nil {
return errors.NotFoundError(fmt.Errorf("The tag %v:%v is not found", art.Repository, art.Tag))
}
w.Header().Set(dockerContentDigest, dig)
w.Header().Set(etag, dig)
go func() {
// After docker 20.10 or containerd, the client heads the tag first,
// Then GET the image by digest, in order to associate the tag with the digest
// Ensure tag after head request, make sure tags in proxy cache keep update
bCtx := orm.Context()
for i := 0; i < ensureTagMaxRetry; i++ {
time.Sleep(ensureTagInterval)
bArt := lib.ArtifactInfo{ProjectName: art.ProjectName, Repository: art.Repository, Digest: string(desc.Digest)}
err := ctl.EnsureTag(bCtx, bArt, art.Tag)
if err == nil {
return
}
log.Debugf("Failed to ensure tag %+v , error %v", art, err)
}
}()
w.Header().Set(contentType, desc.MediaType)
w.Header().Set(contentLength, fmt.Sprintf("%v", desc.Size))
w.Header().Set(dockerContentDigest, string(desc.Digest))
w.Header().Set(etag, string(desc.Digest))
return nil
}

1
src/testing/controller/controller.go
View File

@ -23,3 +23,4 @@ package controller
//go:generate mockery --case snake --dir ../../controller/scanner --name Controller --output ./scanner --outpkg scanner
//go:generate mockery --case snake --dir ../../controller/replication --name Controller --output ./replication --outpkg replication
//go:generate mockery --case snake --dir ../../controller/robot --name Controller --output ./robot --outpkg robot
//go:generate mockery --case snake --dir ../../controller/proxy --name RemoteInterface --output ./proxy --outpkg proxy

106
src/testing/controller/proxy/remote_interface.go Normal file
View File

@ -0,0 +1,106 @@
// Code generated by mockery v2.1.0. DO NOT EDIT.
package proxy
import (
io "io"
distribution "github.com/docker/distribution"
mock "github.com/stretchr/testify/mock"
)
// RemoteInterface is an autogenerated mock type for the RemoteInterface type
type RemoteInterface struct {
mock.Mock
}
// BlobReader provides a mock function with given fields: repo, dig
func (_m *RemoteInterface) BlobReader(repo string, dig string) (int64, io.ReadCloser, error) {
ret := _m.Called(repo, dig)
var r0 int64
if rf, ok := ret.Get(0).(func(string, string) int64); ok {
r0 = rf(repo, dig)
} else {
r0 = ret.Get(0).(int64)
}
var r1 io.ReadCloser
if rf, ok := ret.Get(1).(func(string, string) io.ReadCloser); ok {
r1 = rf(repo, dig)
} else {
if ret.Get(1) != nil {
r1 = ret.Get(1).(io.ReadCloser)
}
}
var r2 error
if rf, ok := ret.Get(2).(func(string, string) error); ok {
r2 = rf(repo, dig)
} else {
r2 = ret.Error(2)
}
return r0, r1, r2
}
// Manifest provides a mock function with given fields: repo, ref
func (_m *RemoteInterface) Manifest(repo string, ref string) (distribution.Manifest, string, error) {
ret := _m.Called(repo, ref)
var r0 distribution.Manifest
if rf, ok := ret.Get(0).(func(string, string) distribution.Manifest); ok {
r0 = rf(repo, ref)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(distribution.Manifest)
}
}
var r1 string
if rf, ok := ret.Get(1).(func(string, string) string); ok {
r1 = rf(repo, ref)
} else {
r1 = ret.Get(1).(string)
}
var r2 error
if rf, ok := ret.Get(2).(func(string, string) error); ok {
r2 = rf(repo, ref)
} else {
r2 = ret.Error(2)
}
return r0, r1, r2
}
// ManifestExist provides a mock function with given fields: repo, ref
func (_m *RemoteInterface) ManifestExist(repo string, ref string) (bool, *distribution.Descriptor, error) {
ret := _m.Called(repo, ref)
var r0 bool
if rf, ok := ret.Get(0).(func(string, string) bool); ok {
r0 = rf(repo, ref)
} else {
r0 = ret.Get(0).(bool)
}
var r1 *distribution.Descriptor
if rf, ok := ret.Get(1).(func(string, string) *distribution.Descriptor); ok {
r1 = rf(repo, ref)
} else {
if ret.Get(1) != nil {
r1 = ret.Get(1).(*distribution.Descriptor)
}
}
var r2 error
if rf, ok := ret.Get(2).(func(string, string) error); ok {
r2 = rf(repo, ref)
} else {
r2 = ret.Error(2)
}
return r0, r1, r2
}

8
src/testing/pkg/registry/client.go
View File

@ -53,9 +53,13 @@ func (f *FakeClient) ListTags(repository string) ([]string, error) {
}
// ManifestExist ...
func (f *FakeClient) ManifestExist(repository, reference string) (bool, string, error) {
func (f *FakeClient) ManifestExist(repository, reference string) (bool, *distribution.Descriptor, error) {
args := f.Called()
return args.Bool(0), args.String(1), args.Error(2)
var desc *distribution.Descriptor
if args[0] != nil {
desc = args[0].(*distribution.Descriptor)
}
return args.Bool(0), desc, args.Error(2)
}
// PullManifest ...