From deaecf2de544ecf088b8cadfb21f97c45a7f932a Mon Sep 17 00:00:00 2001 From: Wang Yan Date: Wed, 22 Mar 2023 21:48:09 +0800 Subject: [PATCH] add subject artifact repo (#18394) add suject_artifact_repo column in the table artifact_accessory Signed-off-by: Wang Yan --- api/v2.0/swagger.yaml | 6 +++++- .../postgresql/0110_2.8.0_schema.up.sql | 5 +++-- src/controller/artifact/controller.go | 2 +- src/controller/artifact/controller_test.go | 2 +- src/pkg/accessory/dao/dao_test.go | 15 +++++++++++++-- src/pkg/accessory/dao/model.go | 1 + src/pkg/accessory/manager.go | 8 ++++++-- src/pkg/accessory/manager_test.go | 2 +- src/pkg/accessory/model/accessory.go | 1 + src/server/middleware/cosign/cosign.go | 1 + src/server/middleware/cosign/cosign_test.go | 1 + src/server/middleware/subject/subject.go | 1 + src/server/middleware/subject/subject_test.go | 1 + src/server/registry/referrers.go | 4 ++-- src/server/registry/referrers_test.go | 6 +++--- src/server/v2.0/handler/model/accessory.go | 1 + src/testing/pkg/accessory/manager.go | 10 +++++----- 17 files changed, 47 insertions(+), 20 deletions(-) diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 319b9303e..b0a212156 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -9385,11 +9385,15 @@ definitions: subject_artifact_id: type: integer format: int64 - description: The subject artifact id of the accessory + description: Going to be deprecated, use repo and digest for insteand. The subject artifact id of the accessory. subject_artifact_digest: type: string description: The subject artifact digest of the accessory x-omitempty: false + subject_artifact_repo: + type: string + description: The subject artifact repository name of the accessory + x-omitempty: false size: type: integer format: int64 diff --git a/make/migrations/postgresql/0110_2.8.0_schema.up.sql b/make/migrations/postgresql/0110_2.8.0_schema.up.sql index a5bdd545c..5dfd7cd59 100644 --- a/make/migrations/postgresql/0110_2.8.0_schema.up.sql +++ b/make/migrations/postgresql/0110_2.8.0_schema.up.sql @@ -1,8 +1,9 @@ /* remove the redundant data from table artifact_blob */ delete from artifact_blob afb where not exists (select digest from blob b where b.digest = afb.digest_af); -/* add subject_artifact_digest*/ +/* add subject_artifact_digest and subject_artifact_repo */ alter table artifact_accessory add column IF NOT EXISTS subject_artifact_digest varchar(1024); +alter table artifact_accessory add column IF NOT EXISTS subject_artifact_repo varchar(1024); DO $$ DECLARE @@ -12,7 +13,7 @@ BEGIN FOR acc IN SELECT * FROM artifact_accessory LOOP SELECT * INTO art from artifact where id = acc.subject_artifact_id; - UPDATE artifact_accessory SET subject_artifact_digest=art.digest WHERE subject_artifact_id = art.id; + UPDATE artifact_accessory SET subject_artifact_digest=art.digest, subject_artifact_repo=art.repository_name WHERE subject_artifact_id = art.id; END LOOP; END $$; diff --git a/src/controller/artifact/controller.go b/src/controller/artifact/controller.go index 040ef2c80..3d240d74c 100644 --- a/src/controller/artifact/controller.go +++ b/src/controller/artifact/controller.go @@ -161,7 +161,7 @@ func (c *controller) Ensure(ctx context.Context, repository, digest string, opti } } for _, acc := range option.Accs { - if err = c.accessoryMgr.Ensure(ctx, artifact.Digest, artifact.ID, acc.ArtifactID, acc.Size, acc.Digest, acc.Type); err != nil { + if err = c.accessoryMgr.Ensure(ctx, artifact.Digest, artifact.RepositoryName, artifact.ID, acc.ArtifactID, acc.Size, acc.Digest, acc.Type); err != nil { return false, 0, err } } diff --git a/src/controller/artifact/controller_test.go b/src/controller/artifact/controller_test.go index b1722db65..9dcbc8f9b 100644 --- a/src/controller/artifact/controller_test.go +++ b/src/controller/artifact/controller_test.go @@ -564,7 +564,7 @@ func (c *controllerTestSuite) TestCopy() { c.artMgr.On("Create", mock.Anything, mock.Anything).Return(int64(1), nil) c.regCli.On("Copy", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) c.tagCtl.On("Ensure").Return(nil) - c.accMgr.On("Ensure", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + c.accMgr.On("Ensure", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) _, err := c.ctl.Copy(orm.NewContext(nil, &ormtesting.FakeOrmer{}), "library/hello-world", "latest", "library/hello-world2") c.Require().Nil(err) } diff --git a/src/pkg/accessory/dao/dao_test.go b/src/pkg/accessory/dao/dao_test.go index 2a77010ff..4a23a7b30 100644 --- a/src/pkg/accessory/dao/dao_test.go +++ b/src/pkg/accessory/dao/dao_test.go @@ -36,6 +36,7 @@ type daoTestSuite struct { artDAO artdao.DAO artifactID int64 subArtifactID int64 + subArtifactRepo string subArtifactDigest string accID int64 ctx context.Context @@ -53,6 +54,7 @@ func (d *daoTestSuite) SetupSuite() { MediaType: "application/vnd.oci.image.config.v1+json", ManifestMediaType: "application/vnd.oci.image.manifest.v1+json", ProjectID: 1, + RepositoryName: "goharbor", RepositoryID: 1000, Digest: d.DigestString(), } @@ -60,6 +62,7 @@ func (d *daoTestSuite) SetupSuite() { d.subArtifactID = artifactID d.Require().Nil(err) d.subArtifactDigest = art.Digest + d.subArtifactRepo = art.RepositoryName d.artDAO = artdao.New() artifactID, err = d.artDAO.Create(d.ctx, &artdao.Artifact{ @@ -68,6 +71,7 @@ func (d *daoTestSuite) SetupSuite() { ManifestMediaType: "application/vnd.oci.image.manifest.v1+json", ProjectID: 1, RepositoryID: 1000, + RepositoryName: "goharbor", Digest: d.DigestString(), }) d.Require().Nil(err) @@ -76,6 +80,7 @@ func (d *daoTestSuite) SetupSuite() { accID, err := d.dao.Create(d.ctx, &Accessory{ ArtifactID: d.artifactID, SubjectArtifactDigest: d.subArtifactDigest, + SubjectArtifactRepo: d.subArtifactRepo, Digest: d.DigestString(), Size: 1234, Type: "cosign.signature", @@ -153,6 +158,7 @@ func (d *daoTestSuite) TestCreate() { // conflict acc := &Accessory{ ArtifactID: d.artifactID, + SubjectArtifactRepo: d.subArtifactRepo, SubjectArtifactDigest: d.subArtifactDigest, Digest: d.DigestString(), Size: 1234, @@ -181,6 +187,7 @@ func (d *daoTestSuite) TestDeleteOfArtifact() { ManifestMediaType: "application/vnd.oci.image.manifest.v1+json", ProjectID: 1, RepositoryID: 1000, + RepositoryName: "goharbor", Digest: d.DigestString(), } subArtID, err := d.artDAO.Create(d.ctx, art) @@ -193,6 +200,7 @@ func (d *daoTestSuite) TestDeleteOfArtifact() { ManifestMediaType: "application/vnd.oci.image.manifest.v1+json", ProjectID: 1, RepositoryID: 1000, + RepositoryName: "goharbor", Digest: d.DigestString(), }) d.Require().Nil(err) @@ -204,6 +212,7 @@ func (d *daoTestSuite) TestDeleteOfArtifact() { ManifestMediaType: "application/vnd.oci.image.manifest.v1+json", ProjectID: 1, RepositoryID: 1000, + RepositoryName: "goharbor", Digest: d.DigestString(), }) d.Require().Nil(err) @@ -212,6 +221,7 @@ func (d *daoTestSuite) TestDeleteOfArtifact() { acc1 := &Accessory{ ArtifactID: artID1, SubjectArtifactDigest: art.Digest, + SubjectArtifactRepo: art.RepositoryName, Digest: d.DigestString(), Size: 1234, Type: "cosign.signature", @@ -222,6 +232,7 @@ func (d *daoTestSuite) TestDeleteOfArtifact() { acc2 := &Accessory{ ArtifactID: artID2, SubjectArtifactDigest: art.Digest, + SubjectArtifactRepo: art.RepositoryName, Digest: d.DigestString(), Size: 1234, Type: "cosign.signature", @@ -242,14 +253,14 @@ func (d *daoTestSuite) TestDeleteOfArtifact() { _, err = d.dao.DeleteAccessories(d.ctx, &q.Query{ Keywords: map[string]interface{}{ - "SubjectArtifactDigest": art.Digest, + "SubjectArtifactDigest": art.Digest, "SubjectArtifactRepo": art.RepositoryName, }, }) d.Require().Nil(err) accs, err = d.dao.List(d.ctx, &q.Query{ Keywords: map[string]interface{}{ - "SubjectArtifactDigest": art.Digest, + "SubjectArtifactDigest": art.Digest, "SubjectArtifactRepo": art.RepositoryName, }, }) d.Require().Nil(err) diff --git a/src/pkg/accessory/dao/model.go b/src/pkg/accessory/dao/model.go index e140cbd54..c9e5e0783 100644 --- a/src/pkg/accessory/dao/model.go +++ b/src/pkg/accessory/dao/model.go @@ -29,6 +29,7 @@ type Accessory struct { ID int64 `orm:"pk;auto;column(id)" json:"id"` ArtifactID int64 `orm:"column(artifact_id)" json:"artifact_id"` SubjectArtifactID int64 `orm:"column(subject_artifact_id)" json:"subject_artifact_id"` + SubjectArtifactRepo string `orm:"column(subject_artifact_repo)" json:"subject_artifact_repo"` SubjectArtifactDigest string `orm:"column(subject_artifact_digest)" json:"subject_artifact_digest"` Type string `orm:"column(type)" json:"type"` Size int64 `orm:"column(size)" json:"size"` diff --git a/src/pkg/accessory/manager.go b/src/pkg/accessory/manager.go index 6836662be..f2bf3edc8 100644 --- a/src/pkg/accessory/manager.go +++ b/src/pkg/accessory/manager.go @@ -38,7 +38,7 @@ var ( // Manager is the only interface of artifact module to provide the management functions for artifacts type Manager interface { // Ensure ... - Ensure(ctx context.Context, subArtDigest string, subArtID, artifactID, size int64, digest, accType string) error + Ensure(ctx context.Context, subArtDigest, subArtRepo string, subArtID, artifactID, size int64, digest, accType string) error // Get the artifact specified by the ID Get(ctx context.Context, id int64) (accessory model.Accessory, err error) // Count returns the total count of accessory according to the query. @@ -66,7 +66,7 @@ type manager struct { dao dao.DAO } -func (m *manager) Ensure(ctx context.Context, subArtDigest string, subArtID, artifactID, size int64, digest, accType string) error { +func (m *manager) Ensure(ctx context.Context, subArtDigest, subArtRepo string, subArtID, artifactID, size int64, digest, accType string) error { accs, err := m.dao.List(ctx, q.New(q.KeyWords{"ArtifactID": artifactID, "Digest": digest})) if err != nil { return err @@ -78,6 +78,7 @@ func (m *manager) Ensure(ctx context.Context, subArtDigest string, subArtID, art acc := model.AccessoryData{ ArtifactID: artifactID, SubArtifactID: subArtID, + SubArtifactRepo: subArtRepo, SubArtifactDigest: subArtDigest, Digest: digest, Size: size, @@ -96,6 +97,7 @@ func (m *manager) Get(ctx context.Context, id int64) (model.Accessory, error) { ID: acc.ID, ArtifactID: acc.ArtifactID, SubArtifactID: acc.SubjectArtifactID, + SubArtifactRepo: acc.SubjectArtifactRepo, SubArtifactDigest: acc.SubjectArtifactDigest, Size: acc.Size, Digest: acc.Digest, @@ -119,6 +121,7 @@ func (m *manager) List(ctx context.Context, query *q.Query) ([]model.Accessory, ID: accD.ID, ArtifactID: accD.ArtifactID, SubArtifactID: accD.SubjectArtifactID, + SubArtifactRepo: accD.SubjectArtifactRepo, SubArtifactDigest: accD.SubjectArtifactDigest, Size: accD.Size, Digest: accD.Digest, @@ -137,6 +140,7 @@ func (m *manager) Create(ctx context.Context, accessory model.AccessoryData) (in acc := &dao.Accessory{ ArtifactID: accessory.ArtifactID, SubjectArtifactID: accessory.SubArtifactID, + SubjectArtifactRepo: accessory.SubArtifactRepo, SubjectArtifactDigest: accessory.SubArtifactDigest, Size: accessory.Size, Digest: accessory.Digest, diff --git a/src/pkg/accessory/manager_test.go b/src/pkg/accessory/manager_test.go index 5f9a930ac..7e436ce46 100644 --- a/src/pkg/accessory/manager_test.go +++ b/src/pkg/accessory/manager_test.go @@ -45,7 +45,7 @@ func (m *managerTestSuite) SetupTest() { func (m *managerTestSuite) TestEnsure() { mock.OnAnything(m.dao, "List").Return([]*dao.Accessory{}, nil) mock.OnAnything(m.dao, "Create").Return(int64(1), nil) - err := m.mgr.Ensure(nil, string(""), int64(1), int64(2), int64(1), "sha256:1234", model.TypeCosignSignature) + err := m.mgr.Ensure(nil, string(""), string(""), int64(1), int64(2), int64(1), "sha256:1234", model.TypeCosignSignature) m.Require().Nil(err) } diff --git a/src/pkg/accessory/model/accessory.go b/src/pkg/accessory/model/accessory.go index 9ca31bf5c..a960a75a2 100644 --- a/src/pkg/accessory/model/accessory.go +++ b/src/pkg/accessory/model/accessory.go @@ -80,6 +80,7 @@ type AccessoryData struct { ID int64 `json:"id"` ArtifactID int64 `json:"artifact_id"` SubArtifactID int64 `json:"subject_artifact_id"` + SubArtifactRepo string `json:"subject_artifact_repo"` SubArtifactDigest string `json:"subject_artifact_digest"` Type string `json:"type"` Size int64 `json:"size"` diff --git a/src/server/middleware/cosign/cosign.go b/src/server/middleware/cosign/cosign.go index 7d0563764..b863dc010 100644 --- a/src/server/middleware/cosign/cosign.go +++ b/src/server/middleware/cosign/cosign.go @@ -112,6 +112,7 @@ func SignatureMiddleware() func(http.Handler) http.Handler { } accData := model.AccessoryData{ ArtifactID: art.ID, + SubArtifactRepo: info.Repository, SubArtifactDigest: fmt.Sprintf("%s:%s", digest.SHA256, subjectArtDigest), Size: art.Size, Digest: art.Digest, diff --git a/src/server/middleware/cosign/cosign_test.go b/src/server/middleware/cosign/cosign_test.go index b6019a417..14094b8e9 100644 --- a/src/server/middleware/cosign/cosign_test.go +++ b/src/server/middleware/cosign/cosign_test.go @@ -151,6 +151,7 @@ func (suite *MiddlewareTestSuite) TestCosignSignature() { suite.Equal(1, len(accs)) suite.Equal(subArtDigest, accs[0].GetData().SubArtifactDigest) suite.Equal(artID, accs[0].GetData().ArtifactID) + suite.Equal(name, accs[0].GetData().SubArtifactRepo) suite.True(accs[0].IsHard()) suite.Equal(model.TypeCosignSignature, accs[0].GetData().Type) }) diff --git a/src/server/middleware/subject/subject.go b/src/server/middleware/subject/subject.go index 9cbb5f699..abd609f5c 100644 --- a/src/server/middleware/subject/subject.go +++ b/src/server/middleware/subject/subject.go @@ -97,6 +97,7 @@ func Middleware() func(http.Handler) http.Handler { } accData := model.AccessoryData{ ArtifactID: art.ID, + SubArtifactRepo: info.Repository, SubArtifactDigest: mf.Subject.Digest.String(), Size: art.Size, Digest: art.Digest, diff --git a/src/server/middleware/subject/subject_test.go b/src/server/middleware/subject/subject_test.go index 959b964a6..001c36cc2 100644 --- a/src/server/middleware/subject/subject_test.go +++ b/src/server/middleware/subject/subject_test.go @@ -157,6 +157,7 @@ func (suite *MiddlewareTestSuite) TestSubject() { suite.Equal(1, len(accs)) suite.Equal(subArtDigest, accs[0].GetData().SubArtifactDigest) suite.Equal(artID, accs[0].GetData().ArtifactID) + suite.Equal(name, accs[0].GetData().SubArtifactRepo) suite.True(accs[0].IsHard()) suite.Equal(model.TypeSubject, accs[0].GetData().Type) }) diff --git a/src/server/registry/referrers.go b/src/server/registry/referrers.go index 8641ea480..ac1762a4c 100644 --- a/src/server/registry/referrers.go +++ b/src/server/registry/referrers.go @@ -60,9 +60,9 @@ func (r *referrersHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } // Query accessories with matching subject artifact digest and artifactType - query := q.New(q.KeyWords{"SubjectArtifactDigest": art.Digest}) + query := q.New(q.KeyWords{"SubjectArtifactDigest": art.Digest, "SubjectArtifactRepo": art.RepositoryName}) if at != "" { - query = q.New(q.KeyWords{"SubjectArtifactDigest": art.Digest, "Type": at}) + query = q.New(q.KeyWords{"SubjectArtifactDigest": art.Digest, "SubjectArtifactRepo": art.RepositoryName, "Type": at}) } total, err := r.accessoryManager.Count(ctx, query) if err != nil { diff --git a/src/server/registry/referrers_test.go b/src/server/registry/referrers_test.go index 93c809146..db574dea2 100644 --- a/src/server/registry/referrers_test.go +++ b/src/server/registry/referrers_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" beegocontext "github.com/beego/beego/v2/server/web/context" - "github.com/goharbor/harbor/src/lib/q" accessorymodel "github.com/goharbor/harbor/src/pkg/accessory/model" basemodel "github.com/goharbor/harbor/src/pkg/accessory/model/base" "github.com/goharbor/harbor/src/pkg/artifact" @@ -43,15 +42,16 @@ func TestReferrersHandlerOK(t *testing.T) { }, }, nil) - accessoryMock.On("Count", mock.Anything, q.New(q.KeyWords{"SubjectArtifactDigest": digestVal})). + accessoryMock.On("Count", mock.Anything, mock.Anything). Return(int64(1), nil) - accessoryMock.On("List", mock.Anything, q.New(q.KeyWords{"SubjectArtifactDigest": digestVal})). + accessoryMock.On("List", mock.Anything, mock.Anything). Return([]accessorymodel.Accessory{ &basemodel.Default{ Data: accessorymodel.AccessoryData{ ID: 1, ArtifactID: 2, SubArtifactDigest: digestVal, + SubArtifactRepo: "goharbor", Type: accessorymodel.TypeCosignSignature, }, }, diff --git a/src/server/v2.0/handler/model/accessory.go b/src/server/v2.0/handler/model/accessory.go index 7c0e15748..51630a99b 100644 --- a/src/server/v2.0/handler/model/accessory.go +++ b/src/server/v2.0/handler/model/accessory.go @@ -18,6 +18,7 @@ func (a *Accessory) ToSwagger() *models.Accessory { ID: a.ID, ArtifactID: a.ArtifactID, SubjectArtifactID: a.SubArtifactID, + SubjectArtifactRepo: a.SubArtifactRepo, SubjectArtifactDigest: a.SubArtifactDigest, Size: a.Size, Digest: a.Digest, diff --git a/src/testing/pkg/accessory/manager.go b/src/testing/pkg/accessory/manager.go index 397d1d5ad..fd9ebb99f 100644 --- a/src/testing/pkg/accessory/manager.go +++ b/src/testing/pkg/accessory/manager.go @@ -92,13 +92,13 @@ func (_m *Manager) DeleteAccessories(ctx context.Context, _a1 *q.Query) error { return r0 } -// Ensure provides a mock function with given fields: ctx, subArtDigest, subArtID, artifactID, size, digest, accType -func (_m *Manager) Ensure(ctx context.Context, subArtDigest string, subArtID int64, artifactID int64, size int64, digest string, accType string) error { - ret := _m.Called(ctx, subArtDigest, subArtID, artifactID, size, digest, accType) +// Ensure provides a mock function with given fields: ctx, subArtDigest, subArtRepo, subArtID, artifactID, size, digest, accType +func (_m *Manager) Ensure(ctx context.Context, subArtDigest string, subArtRepo string, subArtID int64, artifactID int64, size int64, digest string, accType string) error { + ret := _m.Called(ctx, subArtDigest, subArtRepo, subArtID, artifactID, size, digest, accType) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, int64, int64, int64, string, string) error); ok { - r0 = rf(ctx, subArtDigest, subArtID, artifactID, size, digest, accType) + if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int64, int64, string, string) error); ok { + r0 = rf(ctx, subArtDigest, subArtRepo, subArtID, artifactID, size, digest, accType) } else { r0 = ret.Error(0) }