Upstream
/
harbor
mirror of https://github.com/goharbor/harbor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

update referrer manifest descriptor size (#20207) 

cache manifest when first time pull if cacheEnabled

Signed-off-by: yminer <yminer@vmware.com>
This commit is contained in:
MinerYang 2024-04-09 16:50:46 +08:00 committed by GitHub
parent 461a5fa50d
commit 03d9575d84
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 177 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
src/server/registry/referrers.go
View File

@ -22,11 +22,16 @@ import (
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/goharbor/harbor/src/lib/cache"
"github.com/goharbor/harbor/src/lib/config"
"github.com/goharbor/harbor/src/lib/errors"
lib_http "github.com/goharbor/harbor/src/lib/http"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/accessory"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/cached/manifest/redis"
"github.com/goharbor/harbor/src/pkg/registry"
"github.com/goharbor/harbor/src/server/router"
"github.com/goharbor/harbor/src/server/v2.0/handler"
)
@ -38,12 +43,16 @@ func newReferrersHandler() http.Handler {
return &referrersHandler{
artifactManager: artifact.NewManager(),
accessoryManager: accessory.NewManager(),
registryClient: registry.Cli,
maniCacheManager: redis.NewManager(),
}
}
type referrersHandler struct {
artifactManager artifact.Manager
accessoryManager accessory.Manager
registryClient registry.Client
maniCacheManager redis.CachedManager
}
func (r *referrersHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
@ -75,18 +84,56 @@ func (r *referrersHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
lib_http.SendError(w, err)
return
}
// Build index manifest from accessories
mfs := make([]ocispec.Descriptor, 0)
for _, acc := range accs {
accArt, err := r.artifactManager.GetByDigest(ctx, repository, acc.GetData().Digest)
accArtDigest := acc.GetData().Digest
accArt, err := r.artifactManager.GetByDigest(ctx, repository, accArtDigest)
if err != nil {
lib_http.SendError(w, err)
return
}
mf := ocispec.Descriptor{
// whether get manifest from cache
fromCache := false
// whether need write manifest to cache
writeCache := false
var maniContent []byte
// pull manifest, will try to pull from cache first
// and write to cache when pull manifest from registry at first time
if config.CacheEnabled() {
maniContent, err = r.maniCacheManager.Get(req.Context(), accArtDigest)
if err == nil {
fromCache = true
} else {
log.Debugf("failed to get manifest %s from cache, will fallback to registry, error: %v", accArtDigest, err)
if errors.As(err, &cache.ErrNotFound) {
writeCache = true
}
}
}
if !fromCache {
mani, _, err := r.registryClient.PullManifest(accArt.RepositoryName, accArtDigest)
if err != nil {
lib_http.SendError(w, err)
return
}
_, maniContent, err = mani.Payload()
if err != nil {
lib_http.SendError(w, err)
return
}
// write manifest to cache when first time pulling
if writeCache {
err = r.maniCacheManager.Save(req.Context(), accArtDigest, maniContent)
if err != nil {
log.Warningf("failed to save accArt manifest %s to cache, error: %v", accArtDigest, err)
}
}
}
desc := ocispec.Descriptor{
MediaType: accArt.ManifestMediaType,
Size: accArt.Size,
Size: int64(len(maniContent)),
Digest: digest.Digest(accArt.Digest),
Annotations: accArt.Annotations,
ArtifactType: accArt.ArtifactType,
@ -94,10 +141,10 @@ func (r *referrersHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// filter use accArt.ArtifactType as artifactType
if at != "" {
if accArt.ArtifactType == at {
mfs = append(mfs, mf)
mfs = append(mfs, desc)
}
} else {
mfs = append(mfs, mf)
mfs = append(mfs, desc)
}
}

129
src/server/registry/referrers_test.go
View File

@ -3,20 +3,50 @@ package registry
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
beegocontext "github.com/beego/beego/v2/server/web/context"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/stretchr/testify/assert"
"github.com/goharbor/harbor/src/lib/cache"
"github.com/goharbor/harbor/src/lib/config"
accessorymodel "github.com/goharbor/harbor/src/pkg/accessory/model"
basemodel "github.com/goharbor/harbor/src/pkg/accessory/model/base"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/server/router"
"github.com/goharbor/harbor/src/testing/mock"
accessorytesting "github.com/goharbor/harbor/src/testing/pkg/accessory"
arttesting "github.com/goharbor/harbor/src/testing/pkg/artifact"
testmanifest "github.com/goharbor/harbor/src/testing/pkg/cached/manifest/redis"
regtesting "github.com/goharbor/harbor/src/testing/pkg/registry"
)
var (
OCIManifest = `{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"config": {
"mediaType": "application/vnd.example.sbom",
"digest": "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03",
"size": 123
},
"layers": [
{
"mediaType": "application/vnd.example.data.v1.tar+gzip",
"digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317",
"size": 1234
}
],
"annotations": {
"name": "test-image"
}
}`
)
func TestReferrersHandlerOK(t *testing.T) {
@ -35,10 +65,10 @@ func TestReferrersHandlerOK(t *testing.T) {
artifactMock.On("GetByDigest", mock.Anything, mock.Anything, mock.Anything).
Return(&artifact.Artifact{
Digest: digestVal,
Digest: "sha256:4911bb745e19a6b5513755f3d033f10ef10c34b40edc631809e28be8a7c005f6",
ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
MediaType: "application/vnd.example.sbom",
ArtifactType: "application/vnd.example+type",
ArtifactType: "application/vnd.example.sbom",
Size: 1000,
Annotations: map[string]string{
"name": "test-image",
@ -56,13 +86,23 @@ func TestReferrersHandlerOK(t *testing.T) {
SubArtifactDigest: digestVal,
SubArtifactRepo: "goharbor",
Type: accessorymodel.TypeCosignSignature,
Digest: "sha256:4911bb745e19a6b5513755f3d033f10ef10c34b40edc631809e28be8a7c005f6",
},
},
}, nil)
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifest))
if err != nil {
t.Fatal(err)
}
regCliMock := &regtesting.Client{}
config.DefaultMgr().Set(context.TODO(), "cache_enabled", false)
mock.OnAnything(regCliMock, "PullManifest").Return(manifest, "", nil)
handler := &referrersHandler{
artifactManager: artifactMock,
accessoryManager: accessoryMock,
registryClient: regCliMock,
}
handler.ServeHTTP(rec, req)
@ -72,10 +112,89 @@ func TestReferrersHandlerOK(t *testing.T) {
t.Errorf("Expected status code %d, but got %d", http.StatusOK, rec.Code)
}
index := &ocispec.Index{}
json.Unmarshal([]byte(rec.Body.String()), index)
if index.Manifests[0].ArtifactType != "application/vnd.example+type" {
t.Errorf("Expected response body %s, but got %s", "application/vnd.example+type", rec.Body.String())
json.Unmarshal(rec.Body.Bytes(), index)
if index.Manifests[0].ArtifactType != "application/vnd.example.sbom" {
t.Errorf("Expected response body %s, but got %s", "application/vnd.example.sbom", rec.Body.String())
}
_, content, _ := manifest.Payload()
assert.Equal(t, int64(len(content)), index.Manifests[0].Size)
}
func TestReferrersHandlerSavetoCache(t *testing.T) {
rec := httptest.NewRecorder()
digestVal := "sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b"
req, err := http.NewRequest("GET", "/v2/test/repository/referrers/sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b", nil)
if err != nil {
t.Fatal(err)
}
input := &beegocontext.BeegoInput{}
input.SetParam(":reference", digestVal)
*req = *(req.WithContext(context.WithValue(req.Context(), router.ContextKeyInput{}, input)))
artifactMock := &arttesting.Manager{}
accessoryMock := &accessorytesting.Manager{}
artifactMock.On("GetByDigest", mock.Anything, mock.Anything, mock.Anything).
Return(&artifact.Artifact{
Digest: "sha256:4911bb745e19a6b5513755f3d033f10ef10c34b40edc631809e28be8a7c005f6",
ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
MediaType: "application/vnd.example.sbom",
ArtifactType: "application/vnd.example.sbom",
Size: 1000,
Annotations: map[string]string{
"name": "test-image",
},
}, nil)
accessoryMock.On("Count", mock.Anything, mock.Anything).
Return(int64(1), nil)
accessoryMock.On("List", mock.Anything, mock.Anything).
Return([]accessorymodel.Accessory{
&basemodel.Default{
Data: accessorymodel.AccessoryData{
ID: 1,
ArtifactID: 2,
SubArtifactDigest: digestVal,
SubArtifactRepo: "goharbor",
Type: accessorymodel.TypeCosignSignature,
Digest: "sha256:4911bb745e19a6b5513755f3d033f10ef10c34b40edc631809e28be8a7c005f6",
},
},
}, nil)
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifest))
if err != nil {
t.Fatal(err)
}
// cache_enabled pull from cahce
config.DefaultMgr().Set(context.TODO(), "cache_enabled", true)
cacheManagerMock := &testmanifest.CachedManager{}
mock.OnAnything(cacheManagerMock, "Get").Return(nil, fmt.Errorf("unable to do stuff: %w", cache.ErrNotFound))
regCliMock := &regtesting.Client{}
mock.OnAnything(regCliMock, "PullManifest").Return(manifest, "", nil)
mock.OnAnything(cacheManagerMock, "Save").Return(nil)
handler := &referrersHandler{
artifactManager: artifactMock,
accessoryManager: accessoryMock,
registryClient: regCliMock,
maniCacheManager: cacheManagerMock,
}
handler.ServeHTTP(rec, req)
// check that the response has the expected status code (200 OK)
if rec.Code != http.StatusOK {
t.Errorf("Expected status code %d, but got %d", http.StatusOK, rec.Code)
}
index := &ocispec.Index{}
json.Unmarshal(rec.Body.Bytes(), index)
if index.Manifests[0].ArtifactType != "application/vnd.example.sbom" {
t.Errorf("Expected response body %s, but got %s", "application/vnd.example.sbom", rec.Body.String())
}
_, content, _ := manifest.Payload()
assert.Equal(t, int64(len(content)), index.Manifests[0].Size)
}
func TestReferrersHandlerEmpty(t *testing.T) {