support nydus as a accessory (#18953)

Recognize nydus image(with subject) as a kind of accessory and built the releationship with subject manifest

Signed-off-by: wang yan <wangyan@vmware.com>
This commit is contained in:
Wang Yan 2023-07-20 15:54:23 +08:00 committed by GitHub
parent 5e4163b53b
commit ce893635aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 99 additions and 373 deletions

View File

@ -59,6 +59,7 @@ import (
_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/cosign"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/notation"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/nydus"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/subject"
"github.com/goharbor/harbor/src/pkg/audit"
dbCfg "github.com/goharbor/harbor/src/pkg/config/db"

View File

@ -1,164 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package nydus
import (
"context"
"encoding/json"
"io"
"net/http"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/goharbor/harbor/src/controller/artifact"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg/accessory"
"github.com/goharbor/harbor/src/pkg/accessory/model"
"github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/server/middleware"
)
var (
// nydus boostrap layer annotation
nydusBoostrapAnnotation = "containerd.io/snapshot/nydus-bootstrap"
// source artifact digest annotation
sourceDigestAnnotation = "io.goharbor.artifact.v1alpha1.acceleration.source.digest"
)
// NydusAcceleratorMiddleware middleware to record the linkeage of artifact and its accessory
/*
/v2/library/hello-world/manifests/sha256:f54a58bc1aac5ea1a25d796ae155dc228b3f0e11d046ae276b39c4bf2f13d8c4
{
"schemaVersion": 2,
"config": {
"mediaType": "application/vnd.oci.image.config.v1+json",
"digest": "sha256:f7d0778a3c468a5203e95a9efd4d67ecef0d2a04866bb3320f0d5d637812aaee",
"size": 466
},
"layers": [
{
"mediaType": "application/vnd.oci.image.layer.nydus.blob.v1",
"digest": "sha256:fd9923a8e2bdc53747dbba3311be876a1deff4658785830e6030c5a8287acf74 ",
"size": 3011,
"annotations": {
"containerd.io/snapshot/nydus-blob": "true"
}
},
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"digest": "sha256:d49bf6d7db9dac935b99d4c2c846b0d280f550aae62012f888d5a6e3ca59a589",
"size": 459,
"annotations": {
"containerd.io/snapshot/nydus-blob-ids": "[\"fd9923a8e2bdc53747dbba3311be876a1deff4658785830e6030c5a8287acf74\"]",
"containerd.io/snapshot/nydus-bootstrap": "true",
"containerd.io/snapshot/nydus-rafs-version": "5"
}
}
],
"annotations": {
"io.goharbor.artifact.v1alpha1.acceleration.driver.name":"nydus",
"io.goharbor.artifact.v1alpha1.acceleration.driver.version":"5",
"io.goharbor.artifact.v1alpha1.acceleration.source.digest":"sha256:f54a58bc1aac5ea1a25d796ae155dc228b3f0e11d046ae276b39c4bf2f13d8c4"
}
}
*/
func AcceleratorMiddleware() func(http.Handler) http.Handler {
return middleware.AfterResponse(func(w http.ResponseWriter, r *http.Request, statusCode int) error {
if statusCode != http.StatusCreated {
return nil
}
log.Debug("Start NydusAccelerator Middleware")
ctx := r.Context()
logger := log.G(ctx).WithFields(log.Fields{"middleware": "nydus"})
none := lib.ArtifactInfo{}
info := lib.GetArtifactInfo(ctx)
if info == none {
return errors.New("artifactinfo middleware required before this middleware").WithCode(errors.NotFoundCode)
}
if info.Tag == "" {
return nil
}
body, err := io.ReadAll(r.Body)
if err != nil {
return err
}
contentType := r.Header.Get("Content-Type")
manifest, desc, err := distribution.UnmarshalManifest(contentType, body)
if err != nil {
logger.Errorf("unmarshal manifest failed, error: %v", err)
return err
}
var isNydus bool
for _, descriptor := range manifest.References() {
annotationMap := descriptor.Annotations
if _, ok := annotationMap[nydusBoostrapAnnotation]; ok {
isNydus = true
break
}
}
log.Debug("isNydus: ", isNydus)
_, payload, err := manifest.Payload()
if err != nil {
return err
}
mf := &v1.Manifest{}
if err := json.Unmarshal(payload, mf); err != nil {
return err
}
if isNydus {
subjectArt, err := artifact.Ctl.GetByReference(ctx, info.Repository, mf.Annotations[sourceDigestAnnotation], nil)
if err != nil {
logger.Errorf("failed to get subject artifact: %s, error: %v", info.Tag, err)
return err
}
art, err := artifact.Ctl.GetByReference(ctx, info.Repository, desc.Digest.String(), nil)
if err != nil {
logger.Errorf("failed to get nydus accel accelerator: %s, error: %v", desc.Digest.String(), err)
return err
}
if err := orm.WithTransaction(func(ctx context.Context) error {
id, err := accessory.Mgr.Create(ctx, model.AccessoryData{
ArtifactID: art.ID,
SubArtifactDigest: subjectArt.Digest,
Size: desc.Size,
Digest: desc.Digest.String(),
Type: model.TypeNydusAccelerator,
})
log.Debug("accessory id:", id)
return err
})(orm.SetTransactionOpNameToContext(ctx, "tx-create-nydus-accessory")); err != nil {
if !errors.IsConflictErr(err) {
logger.Errorf("failed to create nydus accelerator artifact: %s, error: %v", desc.Digest.String(), err)
return err
}
}
}
return nil
})
}

View File

@ -1,207 +0,0 @@
package nydus
import (
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/stretchr/testify/suite"
"github.com/goharbor/harbor/src/controller/repository"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg"
"github.com/goharbor/harbor/src/pkg/accessory"
accessorymodel "github.com/goharbor/harbor/src/pkg/accessory/model"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/nydus"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/distribution"
htesting "github.com/goharbor/harbor/src/testing"
)
type MiddlewareTestSuite struct {
htesting.Suite
}
func (suite *MiddlewareTestSuite) SetupTest() {
suite.Suite.SetupSuite()
}
func (suite *MiddlewareTestSuite) TearDownTest() {
}
func (suite *MiddlewareTestSuite) prepare(name, ref string) (distribution.Manifest, distribution.Descriptor, *http.Request) {
body := fmt.Sprintf(`
{
"schemaVersion": 2,
"config": {
"mediaType": "application/vnd.oci.image.config.v1+json",
"digest": "sha256:f7d0778a3c468a5203e95a9efd4d67ecef0d2a04866bb3320f0d5d637812aaee",
"size": 466
},
"layers": [
{
"mediaType": "application/vnd.oci.image.layer.nydus.blob.v1",
"digest": "sha256:fd9923a8e2bdc53747dbba3311be876a1deff4658785830e6030c5a8287acf74 ",
"size": 3011,
"annotations": {
"containerd.io/snapshot/nydus-blob": "true"
}
},
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"digest": "sha256:d49bf6d7db9dac935b99d4c2c846b0d280f550aae62012f888d5a6e3ca59a589",
"size": 459,
"annotations": {
"containerd.io/snapshot/nydus-blob-ids": "[\"fd9923a8e2bdc53747dbba3311be876a1deff4658785830e6030c5a8287acf74\"]",
"containerd.io/snapshot/nydus-bootstrap": "true",
"containerd.io/snapshot/nydus-rafs-version": "5"
}
}
],
"annotations": {
"io.goharbor.artifact.v1alpha1.acceleration.driver.name":"nydus",
"io.goharbor.artifact.v1alpha1.acceleration.driver.version":"5",
"io.goharbor.artifact.v1alpha1.acceleration.source.digest":"sha256:f54a58bc1aac5ea1a25d796ae155dc228b3f0e11d046ae276b39c4bf2f13d8c4"
}
}
`)
manifest, descriptor, err := distribution.UnmarshalManifest("application/vnd.oci.image.manifest.v1+json", []byte(body))
suite.Nil(err)
req := suite.NewRequest(http.MethodPut, fmt.Sprintf("/v2/%s/manifests/%s", name, ref), strings.NewReader(body))
req.Header.Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
info := lib.ArtifactInfo{
Repository: name,
Reference: ref,
Tag: "latest-nydus",
Digest: descriptor.Digest.String(),
}
return manifest, descriptor, req.WithContext(lib.WithArtifactInfo(req.Context(), info))
}
func (suite *MiddlewareTestSuite) addArt(pid, repositoryID int64, repositoryName, dgt string) int64 {
af := &artifact.Artifact{
Type: "Docker-Image",
ProjectID: pid,
RepositoryID: repositoryID,
RepositoryName: repositoryName,
Digest: dgt,
Size: 1024,
PushTime: time.Now(),
PullTime: time.Now(),
}
afid, err := pkg.ArtifactMgr.Create(suite.Context(), af)
suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
return afid
}
func (suite *MiddlewareTestSuite) addArtAcc(pid, repositoryID int64, repositoryName, dgt, accdgt string) int64 {
subaf := &artifact.Artifact{
Type: "Docker-Image",
ProjectID: pid,
RepositoryID: repositoryID,
RepositoryName: repositoryName,
Digest: dgt,
Size: 1024,
PushTime: time.Now(),
PullTime: time.Now(),
}
_, err := pkg.ArtifactMgr.Create(suite.Context(), subaf)
suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
af := &artifact.Artifact{
Type: "Nydus",
ProjectID: pid,
RepositoryID: repositoryID,
RepositoryName: repositoryName,
Digest: accdgt,
Size: 1024,
PushTime: time.Now(),
PullTime: time.Now(),
}
afid, err := pkg.ArtifactMgr.Create(suite.Context(), af)
suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
accid, err := accessory.Mgr.Create(suite.Context(), accessorymodel.AccessoryData{
ID: 1,
ArtifactID: afid,
SubArtifactDigest: subaf.Digest,
Digest: accdgt,
Type: accessorymodel.TypeNydusAccelerator,
})
suite.Nil(err, fmt.Sprintf("Add artifact accesspry failed for %d", repositoryID))
return accid
}
func (suite *MiddlewareTestSuite) TestNydusAccelerator() {
suite.WithProject(func(projectID int64, projectName string) {
name := fmt.Sprintf("%s/hello-world", projectName)
subArtDigest := "sha256:f54a58bc1aac5ea1a25d796ae155dc228b3f0e11d046ae276b39c4bf2f13d8c4"
_, descriptor, req := suite.prepare(name, subArtDigest)
// create sunjectArtifact repository
_, repoId, err := repository.Ctl.Ensure(suite.Context(), name)
suite.Nil(err)
// add subject artifact
suite.addArt(projectID, repoId, name, subArtDigest)
// add nydus artifact
artID := suite.addArt(projectID, repoId, name, descriptor.Digest.String())
suite.Nil(err)
res := httptest.NewRecorder()
next := suite.NextHandler(http.StatusCreated, map[string]string{"Docker-Content-Digest": descriptor.Digest.String()})
AcceleratorMiddleware()(next).ServeHTTP(res, req)
suite.Equal(http.StatusCreated, res.Code)
accs, _ := accessory.Mgr.List(suite.Context(), &q.Query{
Keywords: map[string]interface{}{
"SubjectArtifactDigest": subArtDigest,
},
})
suite.Equal(1, len(accs))
suite.Equal(subArtDigest, accs[0].GetData().SubArtifactDigest)
suite.Equal(artID, accs[0].GetData().ArtifactID)
suite.True(accs[0].IsHard())
suite.Equal(accessorymodel.TypeNydusAccelerator, accs[0].GetData().Type)
})
}
func (suite *MiddlewareTestSuite) TestNydusAcceleratorDup() {
suite.WithProject(func(projectID int64, projectName string) {
name := fmt.Sprintf("%s/hello-world", projectName)
subArtDigest := "sha256:f54a58bc1aac5ea1a25d796ae155dc228b3f0e11d046ae276b39c4bf2f13d8c4"
_, descriptor, req := suite.prepare(name, subArtDigest)
_, repoId, err := repository.Ctl.Ensure(suite.Context(), name)
suite.Nil(err)
accID := suite.addArtAcc(projectID, repoId, name, subArtDigest, descriptor.Digest.String())
res := httptest.NewRecorder()
next := suite.NextHandler(http.StatusCreated, map[string]string{"Docker-Content-Digest": descriptor.Digest.String()})
AcceleratorMiddleware()(next).ServeHTTP(res, req)
suite.Equal(http.StatusCreated, res.Code)
accs, _ := accessory.Mgr.List(suite.Context(), &q.Query{
Keywords: map[string]interface{}{
"ID": accID,
},
})
suite.Equal(1, len(accs))
suite.Equal(descriptor.Digest.String(), accs[0].GetData().Digest)
suite.True(accs[0].IsHard())
suite.Equal(accessorymodel.TypeNydusAccelerator, accs[0].GetData().Type)
})
}
func TestMiddlewareTestSuite(t *testing.T) {
suite.Run(t, &MiddlewareTestSuite{})
}

View File

@ -20,6 +20,7 @@ import (
"io"
"net/http"
"github.com/docker/distribution/manifest/schema2"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/goharbor/harbor/src/controller/artifact"
@ -35,6 +36,9 @@ import (
var (
// the media type of notation signature layer
mediaTypeNotationLayer = "application/vnd.cncf.notary.signature"
// annotation of nydus image
layerAnnotationNydusBootstrap = "containerd.io/snapshot/nydus-bootstrap"
)
/*
@ -121,11 +125,14 @@ func Middleware() func(http.Handler) http.Handler {
Size: art.Size,
Digest: art.Digest,
}
accData.Type = model.TypeSubject
switch mf.Config.MediaType {
case schema2.MediaTypeImageConfig:
if isNydusImage(mf) {
accData.Type = model.TypeNydusAccelerator
}
case mediaTypeNotationLayer:
accData.Type = model.TypeNotationSignature
default:
accData.Type = model.TypeSubject
}
if subjectArt != nil {
accData.SubArtifactID = subjectArt.ID
@ -145,3 +152,17 @@ func Middleware() func(http.Handler) http.Handler {
return nil
})
}
// isNydusImage checks if the image is a nydus image.
func isNydusImage(manifest *ocispec.Manifest) bool {
layers := manifest.Layers
if len(layers) != 0 {
desc := layers[len(layers)-1]
if desc.Annotations == nil {
return false
}
_, hasAnno := desc.Annotations[layerAnnotationNydusBootstrap]
return hasAnno
}
return false
}

View File

@ -1,6 +1,7 @@
package subject
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
@ -8,6 +9,7 @@ import (
"testing"
"time"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/stretchr/testify/suite"
"github.com/goharbor/harbor/src/controller/repository"
@ -191,6 +193,79 @@ func (suite *MiddlewareTestSuite) TestSubjectDup() {
})
}
func (suite *MiddlewareTestSuite) TestIsNydusImage() {
mf := `{
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"config": {
"mediaType": "application/vnd.docker.container.image.v1+json",
"digest": "sha256:e314d79415361272a5ff6919ce70eb1d82ae55641ff60dcd8286b731cae2b5e7",
"size": 3322
},
"layers": [
{
"mediaType": "application/vnd.oci.image.layer.nydus.blob.v1",
"digest": "sha256:bce0a563197a6aae0044f2063bf95f43bb956640b374fbdf0886cbc6926e2b7c",
"size": 3440759,
"annotations": {
"containerd.io/snapshot/nydus-blob": "true"
}
},
{
"mediaType": "application/vnd.oci.image.layer.nydus.blob.v1",
"digest": "sha256:7dedc3aaf7177a1d6792efcf1eae1305033fbac8dc48eb0caf49373b5d21475f",
"size": 337049,
"annotations": {
"containerd.io/snapshot/nydus-blob": "true"
}
},
{
"mediaType": "application/vnd.oci.image.layer.nydus.blob.v1",
"digest": "sha256:f6bf79efcfc89f657b9705ef9ed77659e413e355efac8c6d3eea49d908c9218a",
"size": 5810244,
"annotations": {
"containerd.io/snapshot/nydus-blob": "true"
}
},
{
"mediaType": "application/vnd.oci.image.layer.nydus.blob.v1",
"digest": "sha256:35c290e1471c2f546ba7ca8eb47b334c0234e6a2d2b274c54fe96e016c1913c7",
"size": 7936,
"annotations": {
"containerd.io/snapshot/nydus-blob": "true"
}
},
{
"mediaType": "application/vnd.oci.image.layer.nydus.blob.v1",
"digest": "sha256:1f168a347d1c654776644b331e631c3a1208699e2f608e29d8e3fd74e5fd99e8",
"size": 7728,
"annotations": {
"containerd.io/snapshot/nydus-blob": "true"
}
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:86211e9295fabea433b7186ddfa6fd31af048a2f6fe3cf8d747b6f7ea39c0ea6",
"size": 35092,
"annotations": {
"containerd.io/snapshot/nydus-bootstrap": "true",
"containerd.io/snapshot/nydus-fs-version": "6"
}
}
],
"subject": {
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"digest": "sha256:f4d532d482a050a3bb02886be6d6deda9c22cf8df44b1465f04c8648ee573a70",
"size": 1363
}
}`
manifest := &ocispec.Manifest{}
err := json.Unmarshal([]byte(mf), manifest)
suite.Nil(err)
suite.True(isNydusImage(manifest))
}
func TestMiddlewareTestSuite(t *testing.T) {
suite.Run(t, &MiddlewareTestSuite{})
}