add nydus middleware (#17126)

Signed-off-by: yminer <yminer@vmware.com>

remove comments

Signed-off-by: yminer <yminer@vmware.com>

update ut manifest

Signed-off-by: yminer <yminer@vmware.com>

modify comment manifest

Signed-off-by: yminer <yminer@vmware.com>

updtae ut testcase

Signed-off-by: yminer <yminer@vmware.com>

fixwhitespace lint

Signed-off-by: yminer <yminer@vmware.com>

update isNydus judgement && define annotation var

Signed-off-by: yminer <yminer@vmware.com>

whitespace lint

Signed-off-by: yminer <yminer@vmware.com>

src/controller/icon/controller.go

@@ -56,6 +56,10 @@ var (
 			path:   "./icons/cosign.png",
 			resize: false,
 		},
+		icon.DigestOfIconAccNydus: {
+			path:   "./icons/nydus.png",
+			resize: false,
+		},
 		icon.DigestOfIconDefault: {
 			path:   "./icons/default.png",
 			resize: true,

src/core/main.go

@@ -55,6 +55,7 @@ import (
 	"github.com/goharbor/harbor/src/migration"
 	_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
 	_ "github.com/goharbor/harbor/src/pkg/accessory/model/cosign"
+	_ "github.com/goharbor/harbor/src/pkg/accessory/model/nydus"
 	"github.com/goharbor/harbor/src/pkg/audit"
 	dbCfg "github.com/goharbor/harbor/src/pkg/config/db"
 	_ "github.com/goharbor/harbor/src/pkg/config/inmemory"

src/lib/icon/const.go

@@ -10,4 +10,5 @@ const (
 	// ToDo add the accessories images
 	DigestOfIconAccDefault = ""
 	DigestOfIconAccCosign  = "sha256:20401d5b3a0f6dbc607c8d732eb08471af4ae6b19811a4efce8c6a724aed2882"
+	DigestOfIconAccNydus   = "sha256:dfcb6617cd9c144358dc1b305b87bbe34f0b619f1e329116e6aee2e41f2e34cf"
 )

src/pkg/accessory/manager.go

@@ -16,6 +16,7 @@ package accessory
 
 import (
 	"context"
+
 	"github.com/goharbor/harbor/src/lib/errors"
 	"github.com/goharbor/harbor/src/lib/icon"
 	"github.com/goharbor/harbor/src/lib/q"
@@ -30,6 +31,7 @@ var (
 	// icon digests for each known type
 	defaultIcons = map[string]string{
 		model.TypeCosignSignature:  icon.DigestOfIconAccCosign,
+		model.TypeNydusAccelerator: icon.DigestOfIconAccNydus,
 	}
 )
 

src/pkg/accessory/manager_test.go

@@ -15,15 +15,17 @@
 package accessory
 
 import (
+	"testing"
+
 	"github.com/goharbor/harbor/src/lib/q"
 	"github.com/goharbor/harbor/src/pkg/accessory/dao"
 	"github.com/goharbor/harbor/src/pkg/accessory/model"
 	_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
 	_ "github.com/goharbor/harbor/src/pkg/accessory/model/cosign"
+	_ "github.com/goharbor/harbor/src/pkg/accessory/model/nydus"
 	"github.com/goharbor/harbor/src/testing/mock"
 	testingdao "github.com/goharbor/harbor/src/testing/pkg/accessory/dao"
 	"github.com/stretchr/testify/suite"
-	"testing"
 )
 
 type managerTestSuite struct {

src/pkg/accessory/model/accessory.go

@@ -17,9 +17,10 @@ package model
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/goharbor/harbor/src/lib/errors"
 	"sync"
 	"time"
+
+	"github.com/goharbor/harbor/src/lib/errors"
 )
 
 const (
@@ -63,6 +64,9 @@ const (
 	TypeNone = "base"
 	// TypeCosignSignature ...
 	TypeCosignSignature = "signature.cosign"
+
+	// TypeNydusAccelerator ...
+	TypeNydusAccelerator = "accelerator.nydus"
 )
 
 // AccessoryData ...

src/pkg/accessory/model/nydus/nydus.go

@@ -0,0 +1,46 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nydus
+
+import (
+	"github.com/goharbor/harbor/src/pkg/accessory/model"
+	"github.com/goharbor/harbor/src/pkg/accessory/model/base"
+)
+
+// Nydus accelerator model
+type Nydus struct {
+	base.Default
+}
+
+// Kind gives the reference type of nydus accelerator.
+func (ny *Nydus) Kind() string {
+	return model.RefHard
+}
+
+// IsHard ...
+func (ny *Nydus) IsHard() bool {
+	return true
+}
+
+// New returns nydus accelerator
+func New(data model.AccessoryData) model.Accessory {
+	return &Nydus{base.Default{
+		Data: data,
+	}}
+}
+
+func init() {
+	model.Register(model.TypeNydusAccelerator, New)
+}

src/pkg/accessory/model/nydus/nydus_test.go

@@ -0,0 +1,70 @@
+package nydus
+
+import (
+	"testing"
+
+	"github.com/goharbor/harbor/src/pkg/accessory/model"
+	htesting "github.com/goharbor/harbor/src/testing"
+	"github.com/stretchr/testify/suite"
+)
+
+type NydusTestSuite struct {
+	htesting.Suite
+	accessory model.Accessory
+	digest    string
+}
+
+func (suite *NydusTestSuite) SetupSuite() {
+	suite.digest = suite.DigestString()
+	suite.accessory, _ = model.New(model.TypeNydusAccelerator,
+		model.AccessoryData{
+			ArtifactID:    1,
+			SubArtifactID: 2,
+			Size:          4321,
+			Digest:        suite.digest,
+		})
+}
+
+func (suite *NydusTestSuite) TestGetID() {
+	suite.Equal(int64(0), suite.accessory.GetData().ID)
+}
+
+func (suite *NydusTestSuite) TestGetArtID() {
+	suite.Equal(int64(1), suite.accessory.GetData().ArtifactID)
+}
+
+func (suite *NydusTestSuite) TestSubGetArtID() {
+	suite.Equal(int64(2), suite.accessory.GetData().SubArtifactID)
+}
+
+func (suite *NydusTestSuite) TestSubGetSize() {
+	suite.Equal(int64(4321), suite.accessory.GetData().Size)
+}
+
+func (suite *NydusTestSuite) TestSubGetDigest() {
+	suite.Equal(suite.digest, suite.accessory.GetData().Digest)
+}
+
+func (suite *NydusTestSuite) TestSubGetType() {
+	suite.Equal(model.TypeNydusAccelerator, suite.accessory.GetData().Type)
+}
+
+func (suite *NydusTestSuite) TestSubGetRefType() {
+	suite.Equal(model.RefHard, suite.accessory.Kind())
+}
+
+func (suite *NydusTestSuite) TestIsSoft() {
+	suite.False(suite.accessory.IsSoft())
+}
+
+func (suite *NydusTestSuite) TestIsHard() {
+	suite.True(suite.accessory.IsHard())
+}
+
+func (suite *NydusTestSuite) TestDisplay() {
+	suite.False(suite.accessory.Display())
+}
+
+func TestCacheTestSuite(t *testing.T) {
+	suite.Run(t, new(NydusTestSuite))
+}

src/server/middleware/nydus/nydus.go

@@ -0,0 +1,149 @@
+package nydus
+
+import (
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/goharbor/harbor/src/controller/artifact"
+	"github.com/goharbor/harbor/src/lib"
+	"github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/lib/orm"
+	"github.com/goharbor/harbor/src/pkg/accessory"
+	"github.com/goharbor/harbor/src/pkg/accessory/model"
+	"github.com/goharbor/harbor/src/pkg/distribution"
+	"github.com/goharbor/harbor/src/server/middleware"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+var (
+	// nydus boostrap layer annotation
+	nydusBoostrapAnnotation = "containerd.io/snapshot/nydus-bootstrap"
+
+	// source artifact digest annotation
+	sourceDigestAnnotation = "io.goharbor.artifact.v1alpha1.acceleration.source.digest"
+)
+
+// NydusAcceleratorMiddleware middleware to record the linkeage of artifact and its accessory
+/*
+/v2/library/hello-world/manifests/sha256:f54a58bc1aac5ea1a25d796ae155dc228b3f0e11d046ae276b39c4bf2f13d8c4
+{
+    "schemaVersion": 2,
+    "config": {
+      "mediaType": "application/vnd.oci.image.config.v1+json",
+      "digest": "sha256:f7d0778a3c468a5203e95a9efd4d67ecef0d2a04866bb3320f0d5d637812aaee",
+      "size": 466
+    },
+    "layers": [
+      {
+        "mediaType": "application/vnd.oci.image.layer.nydus.blob.v1",
+        "digest": "sha256:fd9923a8e2bdc53747dbba3311be876a1deff4658785830e6030c5a8287acf74 ",
+        "size": 3011,
+        "annotations": {
+          "containerd.io/snapshot/nydus-blob": "true"
+        }
+      },
+      {
+        "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+        "digest": "sha256:d49bf6d7db9dac935b99d4c2c846b0d280f550aae62012f888d5a6e3ca59a589",
+        "size": 459,
+        "annotations": {
+            "containerd.io/snapshot/nydus-blob-ids": "[\"fd9923a8e2bdc53747dbba3311be876a1deff4658785830e6030c5a8287acf74\"]",
+            "containerd.io/snapshot/nydus-bootstrap": "true",
+            "containerd.io/snapshot/nydus-rafs-version": "5"
+        }
+      }
+    ],
+    "annotations": {
+        "io.goharbor.artifact.v1alpha1.acceleration.driver.name":"nydus",
+        "io.goharbor.artifact.v1alpha1.acceleration.driver.version":"5",
+        "io.goharbor.artifact.v1alpha1.acceleration.source.digest":"sha256:f54a58bc1aac5ea1a25d796ae155dc228b3f0e11d046ae276b39c4bf2f13d8c4"
+    }
+}
+
+*/
+func AcceleratorMiddleware() func(http.Handler) http.Handler {
+	return middleware.AfterResponse(func(w http.ResponseWriter, r *http.Request, statusCode int) error {
+		if statusCode != http.StatusCreated {
+			return nil
+		}
+
+		log.Debug("Start NydusAccelerator Middleware")
+		ctx := r.Context()
+		logger := log.G(ctx).WithFields(log.Fields{"middleware": "nydus"})
+
+		none := lib.ArtifactInfo{}
+		info := lib.GetArtifactInfo(ctx)
+		if info == none {
+			return errors.New("artifactinfo middleware required before this middleware").WithCode(errors.NotFoundCode)
+		}
+		if info.Tag == "" {
+			return nil
+		}
+
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			return err
+		}
+
+		contentType := r.Header.Get("Content-Type")
+		manifest, desc, err := distribution.UnmarshalManifest(contentType, body)
+		if err != nil {
+			logger.Errorf("unmarshal manifest failed, error: %v", err)
+			return err
+		}
+
+		var isNydus bool
+		for _, descriptor := range manifest.References() {
+			annotationMap := descriptor.Annotations
+			if _, ok := annotationMap[nydusBoostrapAnnotation]; ok {
+				isNydus = true
+				break
+			}
+		}
+		log.Debug("isNydus: ", isNydus)
+
+		_, payload, err := manifest.Payload()
+		if err != nil {
+			return err
+		}
+		mf := &v1.Manifest{}
+		if err := json.Unmarshal(payload, mf); err != nil {
+			return err
+		}
+
+		if isNydus {
+			subjectArt, err := artifact.Ctl.GetByReference(ctx, info.Repository, mf.Annotations[sourceDigestAnnotation], nil)
+			if err != nil {
+				logger.Errorf("failed to get subject artifact: %s, error: %v", info.Tag, err)
+				return err
+			}
+			art, err := artifact.Ctl.GetByReference(ctx, info.Repository, desc.Digest.String(), nil)
+			if err != nil {
+				logger.Errorf("failed to get nydus accel accelerator: %s, error: %v", desc.Digest.String(), err)
+				return err
+			}
+
+			if err := orm.WithTransaction(func(ctx context.Context) error {
+				id, err := accessory.Mgr.Create(ctx, model.AccessoryData{
+					ArtifactID:    art.ID,
+					SubArtifactID: subjectArt.ID,
+					Size:          desc.Size,
+					Digest:        desc.Digest.String(),
+					Type:          model.TypeNydusAccelerator,
+				})
+				log.Debug("accessory id:", id)
+				return err
+			})(orm.SetTransactionOpNameToContext(ctx, "tx-create-nydus-accessory")); err != nil {
+				if !errors.IsConflictErr(err) {
+					logger.Errorf("failed to create nydus accelerator artifact: %s, error: %v", desc.Digest.String(), err)
+					return err
+				}
+			}
+		}
+
+		return nil
+	})
+}

src/server/middleware/nydus/nydus_test.go

@@ -0,0 +1,207 @@
+package nydus
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/goharbor/harbor/src/controller/repository"
+	"github.com/goharbor/harbor/src/lib"
+	"github.com/goharbor/harbor/src/lib/q"
+	"github.com/goharbor/harbor/src/pkg"
+	"github.com/goharbor/harbor/src/pkg/accessory"
+	"github.com/goharbor/harbor/src/pkg/accessory/model"
+	accessorymodel "github.com/goharbor/harbor/src/pkg/accessory/model"
+	_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
+	_ "github.com/goharbor/harbor/src/pkg/accessory/model/nydus"
+	"github.com/goharbor/harbor/src/pkg/artifact"
+	"github.com/goharbor/harbor/src/pkg/distribution"
+	htesting "github.com/goharbor/harbor/src/testing"
+	"github.com/stretchr/testify/suite"
+)
+
+type MiddlewareTestSuite struct {
+	htesting.Suite
+}
+
+func (suite *MiddlewareTestSuite) SetupTest() {
+	suite.Suite.SetupSuite()
+}
+
+func (suite *MiddlewareTestSuite) TearDownTest() {
+}
+
+func (suite *MiddlewareTestSuite) prepare(name, ref string) (distribution.Manifest, distribution.Descriptor, *http.Request) {
+	body := fmt.Sprintf(`
+	{
+		"schemaVersion": 2,
+		"config": {
+		  "mediaType": "application/vnd.oci.image.config.v1+json",
+		  "digest": "sha256:f7d0778a3c468a5203e95a9efd4d67ecef0d2a04866bb3320f0d5d637812aaee",
+		  "size": 466
+		},
+		"layers": [
+		  {
+			"mediaType": "application/vnd.oci.image.layer.nydus.blob.v1",
+			"digest": "sha256:fd9923a8e2bdc53747dbba3311be876a1deff4658785830e6030c5a8287acf74 ",
+			"size": 3011,
+			"annotations": {
+			  "containerd.io/snapshot/nydus-blob": "true"
+			}
+		  },
+		  {
+			"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+			"digest": "sha256:d49bf6d7db9dac935b99d4c2c846b0d280f550aae62012f888d5a6e3ca59a589",
+			"size": 459,
+			"annotations": {
+				"containerd.io/snapshot/nydus-blob-ids": "[\"fd9923a8e2bdc53747dbba3311be876a1deff4658785830e6030c5a8287acf74\"]",
+				"containerd.io/snapshot/nydus-bootstrap": "true",
+				"containerd.io/snapshot/nydus-rafs-version": "5"
+			}
+		  }
+		],
+		"annotations": {
+			"io.goharbor.artifact.v1alpha1.acceleration.driver.name":"nydus",
+			"io.goharbor.artifact.v1alpha1.acceleration.driver.version":"5",
+			"io.goharbor.artifact.v1alpha1.acceleration.source.digest":"sha256:f54a58bc1aac5ea1a25d796ae155dc228b3f0e11d046ae276b39c4bf2f13d8c4"
+		}
+	}
+	`)
+
+	manifest, descriptor, err := distribution.UnmarshalManifest("application/vnd.oci.image.manifest.v1+json", []byte(body))
+	suite.Nil(err)
+
+	req := suite.NewRequest(http.MethodPut, fmt.Sprintf("/v2/%s/manifests/%s", name, ref), strings.NewReader(body))
+	req.Header.Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
+	info := lib.ArtifactInfo{
+		Repository: name,
+		Reference:  ref,
+		Tag:        "latest-nydus",
+		Digest:     descriptor.Digest.String(),
+	}
+
+	return manifest, descriptor, req.WithContext(lib.WithArtifactInfo(req.Context(), info))
+}
+
+func (suite *MiddlewareTestSuite) addArt(pid, repositoryID int64, repositoryName, dgt string) int64 {
+	af := &artifact.Artifact{
+		Type:           "Docker-Image",
+		ProjectID:      pid,
+		RepositoryID:   repositoryID,
+		RepositoryName: repositoryName,
+		Digest:         dgt,
+		Size:           1024,
+		PushTime:       time.Now(),
+		PullTime:       time.Now(),
+	}
+	afid, err := pkg.ArtifactMgr.Create(suite.Context(), af)
+	suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
+	return afid
+}
+
+func (suite *MiddlewareTestSuite) addArtAcc(pid, repositoryID int64, repositoryName, dgt, accdgt string) int64 {
+	subaf := &artifact.Artifact{
+		Type:           "Docker-Image",
+		ProjectID:      pid,
+		RepositoryID:   repositoryID,
+		RepositoryName: repositoryName,
+		Digest:         dgt,
+		Size:           1024,
+		PushTime:       time.Now(),
+		PullTime:       time.Now(),
+	}
+	subafid, err := pkg.ArtifactMgr.Create(suite.Context(), subaf)
+	suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
+
+	af := &artifact.Artifact{
+		Type:           "Nydus",
+		ProjectID:      pid,
+		RepositoryID:   repositoryID,
+		RepositoryName: repositoryName,
+		Digest:         accdgt,
+		Size:           1024,
+		PushTime:       time.Now(),
+		PullTime:       time.Now(),
+	}
+	afid, err := pkg.ArtifactMgr.Create(suite.Context(), af)
+	suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
+
+	accid, err := accessory.Mgr.Create(suite.Context(), accessorymodel.AccessoryData{
+		ID:            1,
+		ArtifactID:    afid,
+		SubArtifactID: subafid,
+		Digest:        accdgt,
+		Type:          accessorymodel.TypeNydusAccelerator,
+	})
+	suite.Nil(err, fmt.Sprintf("Add artifact accesspry failed for %d", repositoryID))
+	return accid
+}
+
+func (suite *MiddlewareTestSuite) TestNydusAccelerator() {
+	suite.WithProject(func(projectID int64, projectName string) {
+		name := fmt.Sprintf("%s/hello-world", projectName)
+		subArtDigest := "sha256:f54a58bc1aac5ea1a25d796ae155dc228b3f0e11d046ae276b39c4bf2f13d8c4"
+		_, descriptor, req := suite.prepare(name, subArtDigest)
+
+		// create sunjectArtifact repository
+		_, repoId, err := repository.Ctl.Ensure(suite.Context(), name)
+		suite.Nil(err)
+
+		// add subject artifact
+		subjectArtID := suite.addArt(projectID, repoId, name, subArtDigest)
+
+		// add nydus artifact
+		artID := suite.addArt(projectID, repoId, name, descriptor.Digest.String())
+		suite.Nil(err)
+
+		res := httptest.NewRecorder()
+		next := suite.NextHandler(http.StatusCreated, map[string]string{"Docker-Content-Digest": descriptor.Digest.String()})
+		AcceleratorMiddleware()(next).ServeHTTP(res, req)
+		suite.Equal(http.StatusCreated, res.Code)
+
+		accs, _ := accessory.Mgr.List(suite.Context(), &q.Query{
+			Keywords: map[string]interface{}{
+				"SubjectArtifactID": subjectArtID,
+			},
+		})
+		suite.Equal(1, len(accs))
+		suite.Equal(subjectArtID, accs[0].GetData().SubArtifactID)
+		suite.Equal(artID, accs[0].GetData().ArtifactID)
+		suite.True(accs[0].IsHard())
+		suite.Equal(model.TypeNydusAccelerator, accs[0].GetData().Type)
+	})
+}
+
+func (suite *MiddlewareTestSuite) TestNydusAcceleratorDup() {
+	suite.WithProject(func(projectID int64, projectName string) {
+		name := fmt.Sprintf("%s/hello-world", projectName)
+		subArtDigest := "sha256:f54a58bc1aac5ea1a25d796ae155dc228b3f0e11d046ae276b39c4bf2f13d8c4"
+		_, descriptor, req := suite.prepare(name, subArtDigest)
+
+		_, repoId, err := repository.Ctl.Ensure(suite.Context(), name)
+		suite.Nil(err)
+		accID := suite.addArtAcc(projectID, repoId, name, subArtDigest, descriptor.Digest.String())
+
+		res := httptest.NewRecorder()
+		next := suite.NextHandler(http.StatusCreated, map[string]string{"Docker-Content-Digest": descriptor.Digest.String()})
+		AcceleratorMiddleware()(next).ServeHTTP(res, req)
+		suite.Equal(http.StatusCreated, res.Code)
+
+		accs, _ := accessory.Mgr.List(suite.Context(), &q.Query{
+			Keywords: map[string]interface{}{
+				"ID": accID,
+			},
+		})
+		suite.Equal(1, len(accs))
+		suite.Equal(descriptor.Digest.String(), accs[0].GetData().Digest)
+		suite.True(accs[0].IsHard())
+		suite.Equal(model.TypeNydusAccelerator, accs[0].GetData().Type)
+	})
+}
+
+func TestMiddlewareTestSuite(t *testing.T) {
+	suite.Run(t, &MiddlewareTestSuite{})
+}

src/server/registry/route.go

@@ -22,6 +22,7 @@ import (
 	"github.com/goharbor/harbor/src/server/middleware/cosign"
 	"github.com/goharbor/harbor/src/server/middleware/immutable"
 	"github.com/goharbor/harbor/src/server/middleware/metric"
+	"github.com/goharbor/harbor/src/server/middleware/nydus"
 	"github.com/goharbor/harbor/src/server/middleware/quota"
 	"github.com/goharbor/harbor/src/server/middleware/repoproxy"
 	"github.com/goharbor/harbor/src/server/middleware/v2auth"
@@ -80,6 +81,7 @@ func RegisterRoutes() {
 		Middleware(immutable.Middleware()).
 		Middleware(quota.PutManifestMiddleware()).
 		Middleware(cosign.SignatureMiddleware()).
+		Middleware(nydus.AcceleratorMiddleware()).
 		Middleware(blob.PutManifestMiddleware()).
 		HandlerFunc(putManifest)
 	// blob head