add middleware for artifact with subject (#18369)

As for the distribution spec 1.1, it supports client to push an manifest with subject field. By leverging this fidle, harbor could build up the linkage between the subject artifact and it's accessories.

Signed-off-by: wang yan <wangyan@vmware.com>

src/core/main.go

@@ -56,6 +56,7 @@ import (
 	"github.com/goharbor/harbor/src/migration"
 	_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
 	_ "github.com/goharbor/harbor/src/pkg/accessory/model/cosign"
+	_ "github.com/goharbor/harbor/src/pkg/accessory/model/subject"
 	"github.com/goharbor/harbor/src/pkg/audit"
 	dbCfg "github.com/goharbor/harbor/src/pkg/config/db"
 	_ "github.com/goharbor/harbor/src/pkg/config/inmemory"

src/pkg/accessory/model/accessory.go

@@ -64,11 +64,15 @@ type RefIdentifier interface {
 const (
 	// TypeNone ...
 	TypeNone = "base"
+
 	// TypeCosignSignature ...
 	TypeCosignSignature = "signature.cosign"
 
 	// TypeNydusAccelerator ...
 	TypeNydusAccelerator = "accelerator.nydus"
+
+	// TypeSubject ...
+	TypeSubject = "subject.accessory"
 )
 
 // AccessoryData ...

src/pkg/accessory/model/subject/subject.go

@@ -0,0 +1,46 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package subject
+
+import (
+	"github.com/goharbor/harbor/src/pkg/accessory/model"
+	"github.com/goharbor/harbor/src/pkg/accessory/model/base"
+)
+
+// Subject model
+type Subject struct {
+	base.Default
+}
+
+// Kind gives the reference type of subject.
+func (s *Subject) Kind() string {
+	return model.RefHard
+}
+
+// IsHard ...
+func (s *Subject) IsHard() bool {
+	return true
+}
+
+// New returns subject
+func New(data model.AccessoryData) model.Accessory {
+	return &Subject{base.Default{
+		Data: data,
+	}}
+}
+
+func init() {
+	model.Register(model.TypeSubject, New)
+}

src/pkg/accessory/model/subject/subject_test.go

@@ -0,0 +1,73 @@
+package subject
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+
+	"github.com/goharbor/harbor/src/pkg/accessory/model"
+	htesting "github.com/goharbor/harbor/src/testing"
+)
+
+type SubjectTestSuite struct {
+	htesting.Suite
+	accessory model.Accessory
+	digest    string
+	subDigest string
+}
+
+func (suite *SubjectTestSuite) SetupSuite() {
+	suite.digest = suite.DigestString()
+	suite.subDigest = suite.DigestString()
+	suite.accessory, _ = model.New(model.TypeSubject,
+		model.AccessoryData{
+			ArtifactID:        1,
+			SubArtifactDigest: suite.subDigest,
+			Size:              4321,
+			Digest:            suite.digest,
+		})
+}
+
+func (suite *SubjectTestSuite) TestGetID() {
+	suite.Equal(int64(0), suite.accessory.GetData().ID)
+}
+
+func (suite *SubjectTestSuite) TestGetArtID() {
+	suite.Equal(int64(1), suite.accessory.GetData().ArtifactID)
+}
+
+func (suite *SubjectTestSuite) TestSubGetArtID() {
+	suite.Equal(suite.subDigest, suite.accessory.GetData().SubArtifactDigest)
+}
+
+func (suite *SubjectTestSuite) TestSubGetSize() {
+	suite.Equal(int64(4321), suite.accessory.GetData().Size)
+}
+
+func (suite *SubjectTestSuite) TestSubGetDigest() {
+	suite.Equal(suite.digest, suite.accessory.GetData().Digest)
+}
+
+func (suite *SubjectTestSuite) TestSubGetType() {
+	suite.Equal(model.TypeSubject, suite.accessory.GetData().Type)
+}
+
+func (suite *SubjectTestSuite) TestSubGetRefType() {
+	suite.Equal(model.RefHard, suite.accessory.Kind())
+}
+
+func (suite *SubjectTestSuite) TestIsSoft() {
+	suite.False(suite.accessory.IsSoft())
+}
+
+func (suite *SubjectTestSuite) TestIsHard() {
+	suite.True(suite.accessory.IsHard())
+}
+
+func (suite *SubjectTestSuite) TestDisplay() {
+	suite.False(suite.accessory.Display())
+}
+
+func TestCacheTestSuite(t *testing.T) {
+	suite.Run(t, new(SubjectTestSuite))
+}

src/server/middleware/subject/subject.go

@@ -0,0 +1,115 @@
+package subject
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+
+	"github.com/goharbor/harbor/src/controller/artifact"
+	"github.com/goharbor/harbor/src/lib"
+	"github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/lib/orm"
+	"github.com/goharbor/harbor/src/pkg/accessory"
+	"github.com/goharbor/harbor/src/pkg/accessory/model"
+	"github.com/goharbor/harbor/src/server/middleware"
+)
+
+/*
+	{
+	  "schemaVersion": 2,
+	  "mediaType": "application/vnd.oci.image.manifest.v1+json",
+	  "config": {
+	    "mediaType": "application/vnd.oci.image.config.v1+json",
+	    "size": 7023,
+	    "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7"
+	  },
+	  "layers": [
+	    {
+	      "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+	      "size": 32654,
+	      "digest": "sha256:9834876dcfb05cb167a5c24953eba58c4ac89b1adf57f28f2f9d09af107ee8f0"
+	    },
+	    {
+	      "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+	      "size": 16724,
+	      "digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b"
+	    },
+	    {
+	      "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+	      "size": 73109,
+	      "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736"
+	    }
+	  ],
+	  "subject": {
+	    "mediaType": "application/vnd.oci.image.manifest.v1+json",
+	    "size": 7682,
+	    "digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270"
+	  },
+	  "annotations": {
+	    "com.example.key1": "value1",
+	    "com.example.key2": "value2"
+	  }
+	}
+*/
+func Middleware() func(http.Handler) http.Handler {
+	return middleware.AfterResponse(func(w http.ResponseWriter, r *http.Request, statusCode int) error {
+		if statusCode != http.StatusCreated {
+			return nil
+		}
+
+		ctx := r.Context()
+		logger := log.G(ctx).WithFields(log.Fields{"middleware": "subject"})
+
+		none := lib.ArtifactInfo{}
+		info := lib.GetArtifactInfo(ctx)
+		if info == none {
+			return errors.New("artifactinfo middleware required before this middleware").WithCode(errors.NotFoundCode)
+		}
+
+		body, err := io.ReadAll(r.Body)
+		if err != nil {
+			return err
+		}
+
+		mf := &ocispec.Manifest{}
+		if err := json.Unmarshal(body, mf); err != nil {
+			logger.Errorf("unmarshal manifest failed, error: %v", err)
+			return err
+		}
+
+		if mf.Subject != nil {
+			subjectArt, err := artifact.Ctl.GetByReference(ctx, info.Repository, mf.Subject.Digest.String(), nil)
+			if err != nil {
+				logger.Errorf("failed to get subject artifact: %s, error: %v", mf.Subject.Digest, err)
+				return err
+			}
+			art, err := artifact.Ctl.GetByReference(ctx, info.Repository, info.Reference, nil)
+			if err != nil {
+				logger.Errorf("failed to get artifact with subject field: %s, error: %v", info.Reference, err)
+				return err
+			}
+
+			if err := orm.WithTransaction(func(ctx context.Context) error {
+				_, err := accessory.Mgr.Create(ctx, model.AccessoryData{
+					ArtifactID:        art.ID,
+					SubArtifactDigest: subjectArt.Digest,
+					Size:              art.Size,
+					Digest:            art.Digest,
+					Type:              model.TypeSubject,
+				})
+				return err
+			})(orm.SetTransactionOpNameToContext(ctx, "tx-create-subject-accessory")); err != nil {
+				if !errors.IsConflictErr(err) {
+					logger.Errorf("failed to create subject accessory artifact: %s, error: %v", art.Digest, err)
+					return err
+				}
+			}
+		}
+
+		return nil
+	})
+}

src/server/middleware/subject/subject_test.go

@@ -0,0 +1,195 @@
+package subject
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+
+	"github.com/goharbor/harbor/src/controller/repository"
+	"github.com/goharbor/harbor/src/lib"
+	"github.com/goharbor/harbor/src/lib/q"
+	"github.com/goharbor/harbor/src/pkg"
+	"github.com/goharbor/harbor/src/pkg/accessory"
+	"github.com/goharbor/harbor/src/pkg/accessory/model"
+	accessorymodel "github.com/goharbor/harbor/src/pkg/accessory/model"
+	_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
+	_ "github.com/goharbor/harbor/src/pkg/accessory/model/cosign"
+	_ "github.com/goharbor/harbor/src/pkg/accessory/model/subject"
+	"github.com/goharbor/harbor/src/pkg/artifact"
+	"github.com/goharbor/harbor/src/pkg/distribution"
+	htesting "github.com/goharbor/harbor/src/testing"
+)
+
+type MiddlewareTestSuite struct {
+	htesting.Suite
+}
+
+func (suite *MiddlewareTestSuite) SetupTest() {
+	suite.Suite.SetupSuite()
+}
+
+func (suite *MiddlewareTestSuite) TearDownTest() {
+}
+
+func (suite *MiddlewareTestSuite) prepare(name, subject string) (distribution.Manifest, distribution.Descriptor, *http.Request) {
+	body := fmt.Sprintf(`
+	{
+   "schemaVersion":2,
+   "mediaType":"application/vnd.oci.image.manifest.v1+json",
+   "config":{
+      "mediaType":"application/vnd.example.sbom",
+      "size":2,
+      "digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"
+   },
+   "layers":[
+      {
+         "mediaType":"application/vnd.example.sbom.text",
+         "size":37,
+         "digest":"sha256:45592a729ef6884ea3297e9510d79104f27aeef5f4919b3a921e3abb7f469709"
+      }
+   ],
+   "annotations":{
+      "org.example.sbom.format":"text"
+   },
+   "subject":{
+      "mediaType":"application/vnd.oci.image.manifest.v1+json",
+      "size":419,
+      "digest":"%s"
+   }}`, subject)
+
+	manifest, descriptor, err := distribution.UnmarshalManifest("application/vnd.oci.image.manifest.v1+json", []byte(body))
+	suite.Nil(err)
+
+	req := suite.NewRequest(http.MethodPut, fmt.Sprintf("/v2/%s/manifests/%s", name, descriptor.Digest.String()), strings.NewReader(body))
+	req.Header.Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
+	info := lib.ArtifactInfo{
+		Repository: name,
+		Reference:  descriptor.Digest.String(),
+		Tag:        descriptor.Digest.String(),
+		Digest:     descriptor.Digest.String(),
+	}
+
+	return manifest, descriptor, req.WithContext(lib.WithArtifactInfo(req.Context(), info))
+}
+
+func (suite *MiddlewareTestSuite) addArt(pid, repositoryID int64, repositoryName, dgt string) int64 {
+	af := &artifact.Artifact{
+		Type:           "Docker-Image",
+		ProjectID:      pid,
+		RepositoryID:   repositoryID,
+		RepositoryName: repositoryName,
+		Digest:         dgt,
+		Size:           1024,
+		PushTime:       time.Now(),
+		PullTime:       time.Now(),
+	}
+	afid, err := pkg.ArtifactMgr.Create(suite.Context(), af)
+	suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
+	return afid
+}
+
+func (suite *MiddlewareTestSuite) addArtAcc(pid, repositoryID int64, repositoryName, dgt, accdgt string) int64 {
+	subaf := &artifact.Artifact{
+		Type:           "Docker-Image",
+		ProjectID:      pid,
+		RepositoryID:   repositoryID,
+		RepositoryName: repositoryName,
+		Digest:         dgt,
+		Size:           1024,
+		PushTime:       time.Now(),
+		PullTime:       time.Now(),
+	}
+	_, err := pkg.ArtifactMgr.Create(suite.Context(), subaf)
+	suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
+
+	af := &artifact.Artifact{
+		Type:           "Subject",
+		ProjectID:      pid,
+		RepositoryID:   repositoryID,
+		RepositoryName: repositoryName,
+		Digest:         accdgt,
+		Size:           1024,
+		PushTime:       time.Now(),
+		PullTime:       time.Now(),
+	}
+	afid, err := pkg.ArtifactMgr.Create(suite.Context(), af)
+	suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
+
+	accid, err := accessory.Mgr.Create(suite.Context(), accessorymodel.AccessoryData{
+		ID:                1,
+		ArtifactID:        afid,
+		SubArtifactDigest: subaf.Digest,
+		Digest:            accdgt,
+		Type:              accessorymodel.TypeSubject,
+	})
+	suite.Nil(err, fmt.Sprintf("Add artifact accesspry failed for %d", repositoryID))
+	return accid
+}
+
+func (suite *MiddlewareTestSuite) TestSubject() {
+	suite.WithProject(func(projectID int64, projectName string) {
+		name := fmt.Sprintf("%s/hello-world", projectName)
+		_, repoId, err := repository.Ctl.Ensure(suite.Context(), name)
+
+		subArtDigest := suite.DigestString()
+		suite.addArt(projectID, repoId, name, subArtDigest)
+
+		_, descriptor, req := suite.prepare(name, subArtDigest)
+		suite.Nil(err)
+		artID := suite.addArt(projectID, repoId, name, descriptor.Digest.String())
+		suite.Nil(err)
+
+		res := httptest.NewRecorder()
+		next := suite.NextHandler(http.StatusCreated, map[string]string{"Docker-Content-Digest": descriptor.Digest.String()})
+		Middleware()(next).ServeHTTP(res, req)
+		suite.Equal(http.StatusCreated, res.Code)
+
+		accs, err := accessory.Mgr.List(suite.Context(), &q.Query{
+			Keywords: map[string]interface{}{
+				"SubjectArtifactDigest": subArtDigest,
+			},
+		})
+		suite.Equal(1, len(accs))
+		suite.Equal(subArtDigest, accs[0].GetData().SubArtifactDigest)
+		suite.Equal(artID, accs[0].GetData().ArtifactID)
+		suite.True(accs[0].IsHard())
+		suite.Equal(model.TypeSubject, accs[0].GetData().Type)
+	})
+}
+
+func (suite *MiddlewareTestSuite) TestSubjectDup() {
+	suite.WithProject(func(projectID int64, projectName string) {
+		name := fmt.Sprintf("%s/hello-world", projectName)
+		_, repoId, err := repository.Ctl.Ensure(suite.Context(), name)
+
+		subArtDigest := suite.DigestString()
+		_, descriptor, req := suite.prepare(name, subArtDigest)
+		suite.Nil(err)
+
+		accID := suite.addArtAcc(projectID, repoId, name, subArtDigest, descriptor.Digest.String())
+
+		res := httptest.NewRecorder()
+		next := suite.NextHandler(http.StatusCreated, map[string]string{"Docker-Content-Digest": descriptor.Digest.String()})
+		Middleware()(next).ServeHTTP(res, req)
+		suite.Equal(http.StatusCreated, res.Code)
+
+		accs, err := accessory.Mgr.List(suite.Context(), &q.Query{
+			Keywords: map[string]interface{}{
+				"ID": accID,
+			},
+		})
+		suite.Equal(1, len(accs))
+		suite.Equal(descriptor.Digest.String(), accs[0].GetData().Digest)
+		suite.True(accs[0].IsHard())
+		suite.Equal(model.TypeSubject, accs[0].GetData().Type)
+	})
+}
+
+func TestMiddlewareTestSuite(t *testing.T) {
+	suite.Run(t, &MiddlewareTestSuite{})
+}

src/server/registry/referrers.go

@@ -88,7 +88,7 @@ func (r *referrersHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			Size:         accArt.Size,
 			Digest:       digest.Digest(accArt.Digest),
 			Annotations:  accArt.Annotations,
-			ArtifactType: acc.GetData().Type,
+			ArtifactType: accArt.MediaType,
 		}
 		mfs = append(mfs, mf)
 	}

src/server/registry/referrers_test.go

@@ -36,6 +36,7 @@ func TestReferrersHandlerOK(t *testing.T) {
 		Return(&artifact.Artifact{
 			Digest:            digestVal,
 			ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
+			MediaType:         "application/vnd.example.sbom",
 			Size:              1000,
 			Annotations: map[string]string{
 				"name": "test-image",
@@ -69,8 +70,8 @@ func TestReferrersHandlerOK(t *testing.T) {
 	}
 	index := &ocispec.Index{}
 	json.Unmarshal([]byte(rec.Body.String()), index)
-	if index.Manifests[0].ArtifactType != "signature.cosign" {
-		t.Errorf("Expected response body %s, but got %s", "signature.cosign", rec.Body.String())
+	if index.Manifests[0].ArtifactType != "application/vnd.example.sbom" {
+		t.Errorf("Expected response body %s, but got %s", "application/vnd.example.sbom", rec.Body.String())
 	}
 }
 

src/server/registry/route.go

@@ -24,6 +24,7 @@ import (
 	"github.com/goharbor/harbor/src/server/middleware/metric"
 	"github.com/goharbor/harbor/src/server/middleware/quota"
 	"github.com/goharbor/harbor/src/server/middleware/repoproxy"
+	"github.com/goharbor/harbor/src/server/middleware/subject"
 	"github.com/goharbor/harbor/src/server/middleware/v2auth"
 	"github.com/goharbor/harbor/src/server/middleware/vulnerable"
 	"github.com/goharbor/harbor/src/server/router"
@@ -80,6 +81,7 @@ func RegisterRoutes() {
 		Middleware(immutable.Middleware()).
 		Middleware(quota.PutManifestMiddleware()).
 		Middleware(cosign.SignatureMiddleware()).
+		Middleware(subject.Middleware()).
 		Middleware(blob.PutManifestMiddleware()).
 		HandlerFunc(putManifest)
 	// blob head