add nydus middleware (#17126)

Signed-off-by: yminer <yminer@vmware.com>

remove comments

Signed-off-by: yminer <yminer@vmware.com>

update ut manifest

Signed-off-by: yminer <yminer@vmware.com>

modify comment manifest

Signed-off-by: yminer <yminer@vmware.com>

updtae ut testcase

Signed-off-by: yminer <yminer@vmware.com>

fixwhitespace lint

Signed-off-by: yminer <yminer@vmware.com>

update isNydus judgement && define annotation var

Signed-off-by: yminer <yminer@vmware.com>

whitespace lint

Signed-off-by: yminer <yminer@vmware.com>
This commit is contained in:
MinerYang 2022-07-08 09:56:10 +08:00 committed by GitHub
parent bd8d66c68d
commit efd9632e96
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 491 additions and 3 deletions

BIN
icons/nydus.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

View File

@ -56,6 +56,10 @@ var (
path: "./icons/cosign.png", path: "./icons/cosign.png",
resize: false, resize: false,
}, },
icon.DigestOfIconAccNydus: {
path: "./icons/nydus.png",
resize: false,
},
icon.DigestOfIconDefault: { icon.DigestOfIconDefault: {
path: "./icons/default.png", path: "./icons/default.png",
resize: true, resize: true,

View File

@ -55,6 +55,7 @@ import (
"github.com/goharbor/harbor/src/migration" "github.com/goharbor/harbor/src/migration"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/base" _ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/cosign" _ "github.com/goharbor/harbor/src/pkg/accessory/model/cosign"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/nydus"
"github.com/goharbor/harbor/src/pkg/audit" "github.com/goharbor/harbor/src/pkg/audit"
dbCfg "github.com/goharbor/harbor/src/pkg/config/db" dbCfg "github.com/goharbor/harbor/src/pkg/config/db"
_ "github.com/goharbor/harbor/src/pkg/config/inmemory" _ "github.com/goharbor/harbor/src/pkg/config/inmemory"

View File

@ -10,4 +10,5 @@ const (
// ToDo add the accessories images // ToDo add the accessories images
DigestOfIconAccDefault = "" DigestOfIconAccDefault = ""
DigestOfIconAccCosign = "sha256:20401d5b3a0f6dbc607c8d732eb08471af4ae6b19811a4efce8c6a724aed2882" DigestOfIconAccCosign = "sha256:20401d5b3a0f6dbc607c8d732eb08471af4ae6b19811a4efce8c6a724aed2882"
DigestOfIconAccNydus = "sha256:dfcb6617cd9c144358dc1b305b87bbe34f0b619f1e329116e6aee2e41f2e34cf"
) )

View File

@ -16,6 +16,7 @@ package accessory
import ( import (
"context" "context"
"github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/icon" "github.com/goharbor/harbor/src/lib/icon"
"github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/lib/q"
@ -30,6 +31,7 @@ var (
// icon digests for each known type // icon digests for each known type
defaultIcons = map[string]string{ defaultIcons = map[string]string{
model.TypeCosignSignature: icon.DigestOfIconAccCosign, model.TypeCosignSignature: icon.DigestOfIconAccCosign,
model.TypeNydusAccelerator: icon.DigestOfIconAccNydus,
} }
) )

View File

@ -15,15 +15,17 @@
package accessory package accessory
import ( import (
"testing"
"github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/accessory/dao" "github.com/goharbor/harbor/src/pkg/accessory/dao"
"github.com/goharbor/harbor/src/pkg/accessory/model" "github.com/goharbor/harbor/src/pkg/accessory/model"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/base" _ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/cosign" _ "github.com/goharbor/harbor/src/pkg/accessory/model/cosign"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/nydus"
"github.com/goharbor/harbor/src/testing/mock" "github.com/goharbor/harbor/src/testing/mock"
testingdao "github.com/goharbor/harbor/src/testing/pkg/accessory/dao" testingdao "github.com/goharbor/harbor/src/testing/pkg/accessory/dao"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"testing"
) )
type managerTestSuite struct { type managerTestSuite struct {

View File

@ -17,9 +17,10 @@ package model
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"github.com/goharbor/harbor/src/lib/errors"
"sync" "sync"
"time" "time"
"github.com/goharbor/harbor/src/lib/errors"
) )
const ( const (
@ -63,6 +64,9 @@ const (
TypeNone = "base" TypeNone = "base"
// TypeCosignSignature ... // TypeCosignSignature ...
TypeCosignSignature = "signature.cosign" TypeCosignSignature = "signature.cosign"
// TypeNydusAccelerator ...
TypeNydusAccelerator = "accelerator.nydus"
) )
// AccessoryData ... // AccessoryData ...

View File

@ -0,0 +1,46 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package nydus
import (
"github.com/goharbor/harbor/src/pkg/accessory/model"
"github.com/goharbor/harbor/src/pkg/accessory/model/base"
)
// Nydus accelerator model
type Nydus struct {
base.Default
}
// Kind gives the reference type of nydus accelerator.
func (ny *Nydus) Kind() string {
return model.RefHard
}
// IsHard ...
func (ny *Nydus) IsHard() bool {
return true
}
// New returns nydus accelerator
func New(data model.AccessoryData) model.Accessory {
return &Nydus{base.Default{
Data: data,
}}
}
func init() {
model.Register(model.TypeNydusAccelerator, New)
}

View File

@ -0,0 +1,70 @@
package nydus
import (
"testing"
"github.com/goharbor/harbor/src/pkg/accessory/model"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/stretchr/testify/suite"
)
type NydusTestSuite struct {
htesting.Suite
accessory model.Accessory
digest string
}
func (suite *NydusTestSuite) SetupSuite() {
suite.digest = suite.DigestString()
suite.accessory, _ = model.New(model.TypeNydusAccelerator,
model.AccessoryData{
ArtifactID: 1,
SubArtifactID: 2,
Size: 4321,
Digest: suite.digest,
})
}
func (suite *NydusTestSuite) TestGetID() {
suite.Equal(int64(0), suite.accessory.GetData().ID)
}
func (suite *NydusTestSuite) TestGetArtID() {
suite.Equal(int64(1), suite.accessory.GetData().ArtifactID)
}
func (suite *NydusTestSuite) TestSubGetArtID() {
suite.Equal(int64(2), suite.accessory.GetData().SubArtifactID)
}
func (suite *NydusTestSuite) TestSubGetSize() {
suite.Equal(int64(4321), suite.accessory.GetData().Size)
}
func (suite *NydusTestSuite) TestSubGetDigest() {
suite.Equal(suite.digest, suite.accessory.GetData().Digest)
}
func (suite *NydusTestSuite) TestSubGetType() {
suite.Equal(model.TypeNydusAccelerator, suite.accessory.GetData().Type)
}
func (suite *NydusTestSuite) TestSubGetRefType() {
suite.Equal(model.RefHard, suite.accessory.Kind())
}
func (suite *NydusTestSuite) TestIsSoft() {
suite.False(suite.accessory.IsSoft())
}
func (suite *NydusTestSuite) TestIsHard() {
suite.True(suite.accessory.IsHard())
}
func (suite *NydusTestSuite) TestDisplay() {
suite.False(suite.accessory.Display())
}
func TestCacheTestSuite(t *testing.T) {
suite.Run(t, new(NydusTestSuite))
}

View File

@ -0,0 +1,149 @@
package nydus
import (
"context"
"encoding/json"
"io/ioutil"
"net/http"
"github.com/goharbor/harbor/src/controller/artifact"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg/accessory"
"github.com/goharbor/harbor/src/pkg/accessory/model"
"github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/server/middleware"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
)
var (
// nydus boostrap layer annotation
nydusBoostrapAnnotation = "containerd.io/snapshot/nydus-bootstrap"
// source artifact digest annotation
sourceDigestAnnotation = "io.goharbor.artifact.v1alpha1.acceleration.source.digest"
)
// NydusAcceleratorMiddleware middleware to record the linkeage of artifact and its accessory
/*
/v2/library/hello-world/manifests/sha256:f54a58bc1aac5ea1a25d796ae155dc228b3f0e11d046ae276b39c4bf2f13d8c4
{
"schemaVersion": 2,
"config": {
"mediaType": "application/vnd.oci.image.config.v1+json",
"digest": "sha256:f7d0778a3c468a5203e95a9efd4d67ecef0d2a04866bb3320f0d5d637812aaee",
"size": 466
},
"layers": [
{
"mediaType": "application/vnd.oci.image.layer.nydus.blob.v1",
"digest": "sha256:fd9923a8e2bdc53747dbba3311be876a1deff4658785830e6030c5a8287acf74 ",
"size": 3011,
"annotations": {
"containerd.io/snapshot/nydus-blob": "true"
}
},
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"digest": "sha256:d49bf6d7db9dac935b99d4c2c846b0d280f550aae62012f888d5a6e3ca59a589",
"size": 459,
"annotations": {
"containerd.io/snapshot/nydus-blob-ids": "[\"fd9923a8e2bdc53747dbba3311be876a1deff4658785830e6030c5a8287acf74\"]",
"containerd.io/snapshot/nydus-bootstrap": "true",
"containerd.io/snapshot/nydus-rafs-version": "5"
}
}
],
"annotations": {
"io.goharbor.artifact.v1alpha1.acceleration.driver.name":"nydus",
"io.goharbor.artifact.v1alpha1.acceleration.driver.version":"5",
"io.goharbor.artifact.v1alpha1.acceleration.source.digest":"sha256:f54a58bc1aac5ea1a25d796ae155dc228b3f0e11d046ae276b39c4bf2f13d8c4"
}
}
*/
func AcceleratorMiddleware() func(http.Handler) http.Handler {
return middleware.AfterResponse(func(w http.ResponseWriter, r *http.Request, statusCode int) error {
if statusCode != http.StatusCreated {
return nil
}
log.Debug("Start NydusAccelerator Middleware")
ctx := r.Context()
logger := log.G(ctx).WithFields(log.Fields{"middleware": "nydus"})
none := lib.ArtifactInfo{}
info := lib.GetArtifactInfo(ctx)
if info == none {
return errors.New("artifactinfo middleware required before this middleware").WithCode(errors.NotFoundCode)
}
if info.Tag == "" {
return nil
}
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return err
}
contentType := r.Header.Get("Content-Type")
manifest, desc, err := distribution.UnmarshalManifest(contentType, body)
if err != nil {
logger.Errorf("unmarshal manifest failed, error: %v", err)
return err
}
var isNydus bool
for _, descriptor := range manifest.References() {
annotationMap := descriptor.Annotations
if _, ok := annotationMap[nydusBoostrapAnnotation]; ok {
isNydus = true
break
}
}
log.Debug("isNydus: ", isNydus)
_, payload, err := manifest.Payload()
if err != nil {
return err
}
mf := &v1.Manifest{}
if err := json.Unmarshal(payload, mf); err != nil {
return err
}
if isNydus {
subjectArt, err := artifact.Ctl.GetByReference(ctx, info.Repository, mf.Annotations[sourceDigestAnnotation], nil)
if err != nil {
logger.Errorf("failed to get subject artifact: %s, error: %v", info.Tag, err)
return err
}
art, err := artifact.Ctl.GetByReference(ctx, info.Repository, desc.Digest.String(), nil)
if err != nil {
logger.Errorf("failed to get nydus accel accelerator: %s, error: %v", desc.Digest.String(), err)
return err
}
if err := orm.WithTransaction(func(ctx context.Context) error {
id, err := accessory.Mgr.Create(ctx, model.AccessoryData{
ArtifactID: art.ID,
SubArtifactID: subjectArt.ID,
Size: desc.Size,
Digest: desc.Digest.String(),
Type: model.TypeNydusAccelerator,
})
log.Debug("accessory id:", id)
return err
})(orm.SetTransactionOpNameToContext(ctx, "tx-create-nydus-accessory")); err != nil {
if !errors.IsConflictErr(err) {
logger.Errorf("failed to create nydus accelerator artifact: %s, error: %v", desc.Digest.String(), err)
return err
}
}
}
return nil
})
}

View File

@ -0,0 +1,207 @@
package nydus
import (
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/goharbor/harbor/src/controller/repository"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg"
"github.com/goharbor/harbor/src/pkg/accessory"
"github.com/goharbor/harbor/src/pkg/accessory/model"
accessorymodel "github.com/goharbor/harbor/src/pkg/accessory/model"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/nydus"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/distribution"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/stretchr/testify/suite"
)
type MiddlewareTestSuite struct {
htesting.Suite
}
func (suite *MiddlewareTestSuite) SetupTest() {
suite.Suite.SetupSuite()
}
func (suite *MiddlewareTestSuite) TearDownTest() {
}
func (suite *MiddlewareTestSuite) prepare(name, ref string) (distribution.Manifest, distribution.Descriptor, *http.Request) {
body := fmt.Sprintf(`
{
"schemaVersion": 2,
"config": {
"mediaType": "application/vnd.oci.image.config.v1+json",
"digest": "sha256:f7d0778a3c468a5203e95a9efd4d67ecef0d2a04866bb3320f0d5d637812aaee",
"size": 466
},
"layers": [
{
"mediaType": "application/vnd.oci.image.layer.nydus.blob.v1",
"digest": "sha256:fd9923a8e2bdc53747dbba3311be876a1deff4658785830e6030c5a8287acf74 ",
"size": 3011,
"annotations": {
"containerd.io/snapshot/nydus-blob": "true"
}
},
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"digest": "sha256:d49bf6d7db9dac935b99d4c2c846b0d280f550aae62012f888d5a6e3ca59a589",
"size": 459,
"annotations": {
"containerd.io/snapshot/nydus-blob-ids": "[\"fd9923a8e2bdc53747dbba3311be876a1deff4658785830e6030c5a8287acf74\"]",
"containerd.io/snapshot/nydus-bootstrap": "true",
"containerd.io/snapshot/nydus-rafs-version": "5"
}
}
],
"annotations": {
"io.goharbor.artifact.v1alpha1.acceleration.driver.name":"nydus",
"io.goharbor.artifact.v1alpha1.acceleration.driver.version":"5",
"io.goharbor.artifact.v1alpha1.acceleration.source.digest":"sha256:f54a58bc1aac5ea1a25d796ae155dc228b3f0e11d046ae276b39c4bf2f13d8c4"
}
}
`)
manifest, descriptor, err := distribution.UnmarshalManifest("application/vnd.oci.image.manifest.v1+json", []byte(body))
suite.Nil(err)
req := suite.NewRequest(http.MethodPut, fmt.Sprintf("/v2/%s/manifests/%s", name, ref), strings.NewReader(body))
req.Header.Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
info := lib.ArtifactInfo{
Repository: name,
Reference: ref,
Tag: "latest-nydus",
Digest: descriptor.Digest.String(),
}
return manifest, descriptor, req.WithContext(lib.WithArtifactInfo(req.Context(), info))
}
func (suite *MiddlewareTestSuite) addArt(pid, repositoryID int64, repositoryName, dgt string) int64 {
af := &artifact.Artifact{
Type: "Docker-Image",
ProjectID: pid,
RepositoryID: repositoryID,
RepositoryName: repositoryName,
Digest: dgt,
Size: 1024,
PushTime: time.Now(),
PullTime: time.Now(),
}
afid, err := pkg.ArtifactMgr.Create(suite.Context(), af)
suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
return afid
}
func (suite *MiddlewareTestSuite) addArtAcc(pid, repositoryID int64, repositoryName, dgt, accdgt string) int64 {
subaf := &artifact.Artifact{
Type: "Docker-Image",
ProjectID: pid,
RepositoryID: repositoryID,
RepositoryName: repositoryName,
Digest: dgt,
Size: 1024,
PushTime: time.Now(),
PullTime: time.Now(),
}
subafid, err := pkg.ArtifactMgr.Create(suite.Context(), subaf)
suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
af := &artifact.Artifact{
Type: "Nydus",
ProjectID: pid,
RepositoryID: repositoryID,
RepositoryName: repositoryName,
Digest: accdgt,
Size: 1024,
PushTime: time.Now(),
PullTime: time.Now(),
}
afid, err := pkg.ArtifactMgr.Create(suite.Context(), af)
suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
accid, err := accessory.Mgr.Create(suite.Context(), accessorymodel.AccessoryData{
ID: 1,
ArtifactID: afid,
SubArtifactID: subafid,
Digest: accdgt,
Type: accessorymodel.TypeNydusAccelerator,
})
suite.Nil(err, fmt.Sprintf("Add artifact accesspry failed for %d", repositoryID))
return accid
}
func (suite *MiddlewareTestSuite) TestNydusAccelerator() {
suite.WithProject(func(projectID int64, projectName string) {
name := fmt.Sprintf("%s/hello-world", projectName)
subArtDigest := "sha256:f54a58bc1aac5ea1a25d796ae155dc228b3f0e11d046ae276b39c4bf2f13d8c4"
_, descriptor, req := suite.prepare(name, subArtDigest)
// create sunjectArtifact repository
_, repoId, err := repository.Ctl.Ensure(suite.Context(), name)
suite.Nil(err)
// add subject artifact
subjectArtID := suite.addArt(projectID, repoId, name, subArtDigest)
// add nydus artifact
artID := suite.addArt(projectID, repoId, name, descriptor.Digest.String())
suite.Nil(err)
res := httptest.NewRecorder()
next := suite.NextHandler(http.StatusCreated, map[string]string{"Docker-Content-Digest": descriptor.Digest.String()})
AcceleratorMiddleware()(next).ServeHTTP(res, req)
suite.Equal(http.StatusCreated, res.Code)
accs, _ := accessory.Mgr.List(suite.Context(), &q.Query{
Keywords: map[string]interface{}{
"SubjectArtifactID": subjectArtID,
},
})
suite.Equal(1, len(accs))
suite.Equal(subjectArtID, accs[0].GetData().SubArtifactID)
suite.Equal(artID, accs[0].GetData().ArtifactID)
suite.True(accs[0].IsHard())
suite.Equal(model.TypeNydusAccelerator, accs[0].GetData().Type)
})
}
func (suite *MiddlewareTestSuite) TestNydusAcceleratorDup() {
suite.WithProject(func(projectID int64, projectName string) {
name := fmt.Sprintf("%s/hello-world", projectName)
subArtDigest := "sha256:f54a58bc1aac5ea1a25d796ae155dc228b3f0e11d046ae276b39c4bf2f13d8c4"
_, descriptor, req := suite.prepare(name, subArtDigest)
_, repoId, err := repository.Ctl.Ensure(suite.Context(), name)
suite.Nil(err)
accID := suite.addArtAcc(projectID, repoId, name, subArtDigest, descriptor.Digest.String())
res := httptest.NewRecorder()
next := suite.NextHandler(http.StatusCreated, map[string]string{"Docker-Content-Digest": descriptor.Digest.String()})
AcceleratorMiddleware()(next).ServeHTTP(res, req)
suite.Equal(http.StatusCreated, res.Code)
accs, _ := accessory.Mgr.List(suite.Context(), &q.Query{
Keywords: map[string]interface{}{
"ID": accID,
},
})
suite.Equal(1, len(accs))
suite.Equal(descriptor.Digest.String(), accs[0].GetData().Digest)
suite.True(accs[0].IsHard())
suite.Equal(model.TypeNydusAccelerator, accs[0].GetData().Type)
})
}
func TestMiddlewareTestSuite(t *testing.T) {
suite.Run(t, &MiddlewareTestSuite{})
}

View File

@ -22,6 +22,7 @@ import (
"github.com/goharbor/harbor/src/server/middleware/cosign" "github.com/goharbor/harbor/src/server/middleware/cosign"
"github.com/goharbor/harbor/src/server/middleware/immutable" "github.com/goharbor/harbor/src/server/middleware/immutable"
"github.com/goharbor/harbor/src/server/middleware/metric" "github.com/goharbor/harbor/src/server/middleware/metric"
"github.com/goharbor/harbor/src/server/middleware/nydus"
"github.com/goharbor/harbor/src/server/middleware/quota" "github.com/goharbor/harbor/src/server/middleware/quota"
"github.com/goharbor/harbor/src/server/middleware/repoproxy" "github.com/goharbor/harbor/src/server/middleware/repoproxy"
"github.com/goharbor/harbor/src/server/middleware/v2auth" "github.com/goharbor/harbor/src/server/middleware/v2auth"
@ -80,6 +81,7 @@ func RegisterRoutes() {
Middleware(immutable.Middleware()). Middleware(immutable.Middleware()).
Middleware(quota.PutManifestMiddleware()). Middleware(quota.PutManifestMiddleware()).
Middleware(cosign.SignatureMiddleware()). Middleware(cosign.SignatureMiddleware()).
Middleware(nydus.AcceleratorMiddleware()).
Middleware(blob.PutManifestMiddleware()). Middleware(blob.PutManifestMiddleware()).
HandlerFunc(putManifest) HandlerFunc(putManifest)
// blob head // blob head