From a269b4f31cb35676cf268c424b3e7eeb1bfe5c6b Mon Sep 17 00:00:00 2001 From: MinerYang Date: Tue, 12 Mar 2024 21:52:56 +0800 Subject: [PATCH] Update support for artifactType for both manifest and index (#20030) add artifact_type for artifact model to support artifactType filter Signed-off-by: yminer add 2.11 sql schema & update index artifactType omitted Signed-off-by: yminer update UT update migrate sql for artifact_type Signed-off-by: yminer remove debug line --- .../postgresql/0140_2.11.0_schema.up.sql | 31 +++ src/controller/artifact/abstractor.go | 20 +- src/controller/artifact/abstractor_test.go | 177 +++++++++++++++++- .../artifact/processor/default_test.go | 105 ++++++++++- src/go.mod | 2 +- src/go.sum | 4 +- src/pkg/artifact/dao/model.go | 1 + src/pkg/artifact/model.go | 3 + src/pkg/artifact/model_test.go | 4 + src/pkg/blob/dao/dao_test.go | 2 +- src/pkg/blob/manager_test.go | 4 +- src/pkg/securityhub/dao/security_test.go | 12 +- src/server/middleware/subject/subject.go | 38 ++-- src/server/registry/referrers.go | 6 +- src/server/registry/referrers_test.go | 7 +- 15 files changed, 374 insertions(+), 42 deletions(-) create mode 100644 make/migrations/postgresql/0140_2.11.0_schema.up.sql diff --git a/make/migrations/postgresql/0140_2.11.0_schema.up.sql b/make/migrations/postgresql/0140_2.11.0_schema.up.sql new file mode 100644 index 000000000..b43f6072f --- /dev/null +++ b/make/migrations/postgresql/0140_2.11.0_schema.up.sql @@ -0,0 +1,31 @@ +/* +table artifact: + id SERIAL PRIMARY KEY NOT NULL, + type varchar(255) NOT NULL, + media_type varchar(255) NOT NULL, + manifest_media_type varchar(255) NOT NULL, + artifact_type varchar(255) NOT NULL, + project_id int NOT NULL, + repository_id int NOT NULL, + repository_name varchar(255) NOT NULL, + digest varchar(255) NOT NULL, + size bigint, + push_time timestamp default CURRENT_TIMESTAMP, + pull_time timestamp, + extra_attrs text, + annotations jsonb, + CONSTRAINT unique_artifact UNIQUE (repository_id, digest) +*/ + +/* +Add new column artifact_type for artifact table to work with oci-spec v1.1.0 list referrer api +*/ +ALTER TABLE artifact ADD COLUMN artifact_type varchar(255); + +/* +set value for artifact_type +then set column artifact_type as not null +*/ +UPDATE artifact SET artifact_type = media_type; + +ALTER TABLE artifact ALTER COLUMN artifact_type SET NOT NULL; \ No newline at end of file diff --git a/src/controller/artifact/abstractor.go b/src/controller/artifact/abstractor.go index 233008157..bbf75a1fa 100644 --- a/src/controller/artifact/abstractor.go +++ b/src/controller/artifact/abstractor.go @@ -127,10 +127,18 @@ func (a *abstractor) abstractManifestV2Metadata(artifact *artifact.Artifact, con } // use the "manifest.config.mediatype" as the media type of the artifact artifact.MediaType = manifest.Config.MediaType - if manifest.Annotations[wasm.AnnotationVariantKey] == wasm.AnnotationVariantValue || manifest.Annotations[wasm.AnnotationHandlerKey] == wasm.AnnotationHandlerValue { artifact.MediaType = wasm.MediaType } + /* + https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#listing-referrers + For referrers list, if the artifactType is empty or missing in the image manifest, the value of artifactType MUST be set to the config descriptor mediaType value + */ + if manifest.ArtifactType != "" { + artifact.ArtifactType = manifest.ArtifactType + } else { + artifact.ArtifactType = manifest.Config.MediaType + } // set size artifact.Size = int64(len(content)) + manifest.Config.Size @@ -153,6 +161,16 @@ func (a *abstractor) abstractIndexMetadata(ctx context.Context, art *artifact.Ar return err } + /* + https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#listing-referrers + For referrers list, If the artifactType is empty or missing in an index, the artifactType MUST be omitted. + */ + if index.ArtifactType != "" { + art.ArtifactType = index.ArtifactType + } else { + art.ArtifactType = "" + } + // set annotations art.Annotations = index.Annotations diff --git a/src/controller/artifact/abstractor_test.go b/src/controller/artifact/abstractor_test.go index 47b340f04..e7955ed1c 100644 --- a/src/controller/artifact/abstractor_test.go +++ b/src/controller/artifact/abstractor_test.go @@ -15,6 +15,7 @@ package artifact import ( + "context" "testing" "github.com/docker/distribution" @@ -175,7 +176,66 @@ var ( "com.example.key1": "value1" } }` - + OCIManifest = `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.example.config.v1+json", + "digest": "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", + "size": 123 + }, + "layers": [ + { + "mediaType": "application/vnd.example.data.v1.tar+gzip", + "digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317", + "size": 1234 + } + ], + "annotations": { + "com.example.key1": "value1" + } +}` + OCIManifestWithArtifactType = `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "artifactType": "application/vnd.example+type", + "config": { + "mediaType": "application/vnd.example.config.v1+json", + "digest": "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", + "size": 123 + }, + "layers": [ + { + "mediaType": "application/vnd.example.data.v1.tar+gzip", + "digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317", + "size": 1234 + } + ], + "annotations": { + "com.example.key1": "value1" + } +}` + OCIManifestWithEmptyConfig = `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "artifactType": "application/vnd.example+type", + "config": { + "mediaType": "application/vnd.oci.empty.v1+json", + "digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "size": 2 + }, + "layers": [ + { + "mediaType": "application/vnd.example+type", + "digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317", + "size": 1234 + } + ], + "annotations": { + "oci.opencontainers.image.created": "2023-01-02T03:04:05Z", + "com.example.data": "payload" + } +}` index = `{ "schemaVersion": 2, "manifests": [ @@ -202,6 +262,34 @@ var ( "com.example.key1": "value1" } }` + indexWithArtifactType = `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "artifactType": "application/vnd.food.stand", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7143, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + "platform": { + "architecture": "ppc64le", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7682, + "digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270", + "platform": { + "architecture": "amd64", + "os": "linux" + } + } + ], + "annotations": { + "com.example.key1": "value1" + } + }` ) type abstractorTestSuite struct { @@ -267,6 +355,67 @@ func (a *abstractorTestSuite) TestAbstractMetadataOfV2Manifest() { a.Equal("value1", artifact.Annotations["com.example.key1"]) } +// oci-spec v1 +func (a *abstractorTestSuite) TestAbstractMetadataOfOCIManifest() { + manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifest)) + a.Require().Nil(err) + a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil) + artifact := &artifact.Artifact{ + ID: 1, + } + a.processor.On("AbstractMetadata", mock.Anything, mock.Anything, mock.Anything).Return(nil) + err = a.abstractor.AbstractMetadata(context.TODO(), artifact) + a.Require().Nil(err) + a.Assert().Equal(int64(1), artifact.ID) + a.Assert().Equal(v1.MediaTypeImageManifest, artifact.ManifestMediaType) + a.Assert().Equal("application/vnd.example.config.v1+json", artifact.MediaType) + a.Assert().Equal("application/vnd.example.config.v1+json", artifact.ArtifactType) + a.Assert().Equal(int64(1916), artifact.Size) + a.Require().Len(artifact.Annotations, 1) + a.Equal("value1", artifact.Annotations["com.example.key1"]) +} + +// oci-spec v1.1.0 with artifactType +func (a *abstractorTestSuite) TestAbstractMetadataOfOCIManifestWithArtifactType() { + manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithArtifactType)) + a.Require().Nil(err) + a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil) + artifact := &artifact.Artifact{ + ID: 1, + } + a.processor.On("AbstractMetadata", mock.Anything, mock.Anything, mock.Anything).Return(nil) + err = a.abstractor.AbstractMetadata(context.TODO(), artifact) + a.Require().Nil(err) + a.Assert().Equal(int64(1), artifact.ID) + a.Assert().Equal(v1.MediaTypeImageManifest, artifact.ManifestMediaType) + a.Assert().Equal("application/vnd.example.config.v1+json", artifact.MediaType) + a.Assert().Equal("application/vnd.example+type", artifact.ArtifactType) + a.Assert().Equal(int64(1966), artifact.Size) + a.Require().Len(artifact.Annotations, 1) + a.Equal("value1", artifact.Annotations["com.example.key1"]) +} + +// empty config with artifactType +func (a *abstractorTestSuite) TestAbstractMetadataOfV2ManifestWithEmptyConfig() { + // v1.MediaTypeImageManifest + manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithEmptyConfig)) + a.Require().Nil(err) + a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil) + artifact := &artifact.Artifact{ + ID: 1, + } + a.processor.On("AbstractMetadata", mock.Anything, mock.Anything, mock.Anything).Return(nil) + err = a.abstractor.AbstractMetadata(context.TODO(), artifact) + a.Require().Nil(err) + a.Assert().Equal(int64(1), artifact.ID) + a.Assert().Equal(v1.MediaTypeImageManifest, artifact.ManifestMediaType) + a.Assert().Equal(v1.MediaTypeEmptyJSON, artifact.MediaType) + a.Assert().Equal("application/vnd.example+type", artifact.ArtifactType) + a.Assert().Equal(int64(1880), artifact.Size) + a.Require().Len(artifact.Annotations, 2) + a.Equal("payload", artifact.Annotations["com.example.data"]) +} + // OCI index func (a *abstractorTestSuite) TestAbstractMetadataOfIndex() { manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageIndex, []byte(index)) @@ -279,17 +428,41 @@ func (a *abstractorTestSuite) TestAbstractMetadataOfIndex() { artifact := &artifact.Artifact{ ID: 1, } - err = a.abstractor.AbstractMetadata(nil, artifact) + err = a.abstractor.AbstractMetadata(context.TODO(), artifact) a.Require().Nil(err) a.Assert().Equal(int64(1), artifact.ID) a.Assert().Equal(v1.MediaTypeImageIndex, artifact.ManifestMediaType) a.Assert().Equal(v1.MediaTypeImageIndex, artifact.MediaType) + a.Assert().Equal("", artifact.ArtifactType) a.Assert().Equal(int64(668), artifact.Size) a.Require().Len(artifact.Annotations, 1) a.Assert().Equal("value1", artifact.Annotations["com.example.key1"]) a.Len(artifact.References, 2) } +func (a *abstractorTestSuite) TestAbstractMetadataOfIndexWithArtifactType() { + manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageIndex, []byte(indexWithArtifactType)) + a.Require().Nil(err) + a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil) + a.argMgr.On("GetByDigest", mock.Anything, mock.Anything, mock.Anything).Return(&artifact.Artifact{ + ID: 2, + Size: 10, + }, nil) + artifact := &artifact.Artifact{ + ID: 1, + } + err = a.abstractor.AbstractMetadata(context.TODO(), artifact) + a.Require().Nil(err) + a.Assert().Equal(int64(1), artifact.ID) + a.Assert().Equal(v1.MediaTypeImageIndex, artifact.ManifestMediaType) + a.Assert().Equal(v1.MediaTypeImageIndex, artifact.MediaType) + a.Assert().Equal("application/vnd.food.stand", artifact.ArtifactType) + a.Assert().Equal(int64(801), artifact.Size) + a.Require().Len(artifact.Annotations, 1) + a.Assert().Equal("value1", artifact.Annotations["com.example.key1"]) + a.Len(artifact.References, 2) +} + type unknownManifest struct{} func (u *unknownManifest) References() []distribution.Descriptor { diff --git a/src/controller/artifact/processor/default_test.go b/src/controller/artifact/processor/default_test.go index 0f6af7734..489941539 100644 --- a/src/controller/artifact/processor/default_test.go +++ b/src/controller/artifact/processor/default_test.go @@ -117,7 +117,31 @@ var ( } ] }` - v2ManifestWithUnknownConfig = `{ + OCIManifestWithUnknownJsonConfig = `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.exmaple.config.v1+json", + "digest": "sha256:48ef4a53c0770222d9752cd0588431dbda54667046208c79804e34c15c1579cd", + "size": 129 + }, + "layers": [ + { + "mediaType": "application/vnd.example.data.v1.tar+gzip", + "digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317", + "size": 1234 + } + ], + "annotations": { + "com.example.key1": "value1" + } + }` + UnknownJsonConfig = `{ + "author": "yminer", + "architecture": "amd64", + "selfdefined": "true" +}` + OCIManifestWithUnknownConfig = `{ "schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json", "config": { @@ -141,7 +165,30 @@ var ( "newUnspecifiedField": null } }` - unknownConfig = `{NHL Peanut Butter on my NHL bagel}` + UnknownConfig = `{NHL Peanut Butter on my NHL bagel}` + + OCIManifestWithEmptyConfig = `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "artifactType": "application/vnd.example+type", + "config": { + "mediaType": "application/vnd.oci.empty.v1+json", + "digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "size": 2 + }, + "layers": [ + { + "mediaType": "application/vnd.example+type", + "digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317", + "size": 1234 + } + ], + "annotations": { + "oci.opencontainers.image.created": "2023-01-02T03:04:05Z", + "com.example.data": "payload" + } + }` + emptyConfig = `{}` ) type defaultProcessorTestSuite struct { @@ -190,6 +237,12 @@ func (d *defaultProcessorTestSuite) TestGetArtifactType() { typee = processor.GetArtifactType(nil, art) d.Equal("IMAGE", typee) + mediaType = "application/vnd.example.config.v1+json" + art = &artifact.Artifact{MediaType: mediaType} + processor = &defaultProcessor{} + typee = processor.GetArtifactType(nil, art) + d.Equal(ArtifactTypeUnknown, typee) + mediaType = "application/vnd.cncf.helm.chart.config.v1+json" art = &artifact.Artifact{MediaType: mediaType} processor = &defaultProcessor{} @@ -229,19 +282,53 @@ func (d *defaultProcessorTestSuite) TestAbstractMetadata() { d.Len(art.ExtraAttrs, 12) } -func (d *defaultProcessorTestSuite) TestAbstractMetadataWithUnknownConfig() { - manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(v2ManifestWithUnknownConfig)) +func (d *defaultProcessorTestSuite) TestAbstractMetadataOfOCIManifesttWithUnknownJsonConfig() { + manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithUnknownJsonConfig)) d.Require().Nil(err) manifestMediaType, content, err := manifest.Payload() d.Require().Nil(err) - configBlob := io.NopCloser(strings.NewReader(unknownConfig)) - d.regCli.On("PullBlob", mock.Anything, mock.Anything).Return(int64(0), configBlob, nil) - art := &artifact.Artifact{ManifestMediaType: manifestMediaType} - err = d.processor.AbstractMetadata(nil, art, content) + configBlob := io.NopCloser(strings.NewReader(UnknownJsonConfig)) + metadata := map[string]interface{}{} + err = json.NewDecoder(configBlob).Decode(&metadata) + d.Require().Nil(err) + + art := &artifact.Artifact{ManifestMediaType: manifestMediaType, MediaType: "application/vnd.example.config.v1+json"} + + d.regCli.On("PullBlob", mock.Anything, mock.Anything).Return(int64(129), configBlob, nil) + d.parser.On("Parse", context.TODO(), mock.AnythingOfType("*artifact.Artifact"), mock.AnythingOfType("[]byte")).Return(nil) + err = d.processor.AbstractMetadata(context.TODO(), art, content) d.Require().Nil(err) d.Len(art.ExtraAttrs, 0) - d.Len(unknownConfig, 35) + d.NotEqual(art.ExtraAttrs, len(metadata)) + +} + +func (d *defaultProcessorTestSuite) TestAbstractMetadataWithUnknownConfig() { + manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithUnknownConfig)) + d.Require().Nil(err) + manifestMediaType, content, err := manifest.Payload() + d.Require().Nil(err) + + configBlob := io.NopCloser(strings.NewReader(UnknownConfig)) + d.regCli.On("PullBlob", mock.Anything, mock.Anything).Return(int64(0), configBlob, nil) + art := &artifact.Artifact{ManifestMediaType: manifestMediaType, MediaType: "application/vnd.nhl.peanut.butter.bagel"} + err = d.processor.AbstractMetadata(context.TODO(), art, content) + d.Require().Nil(err) + d.Len(art.ExtraAttrs, 0) +} + +func (d *defaultProcessorTestSuite) TestAbstractMetadataWithEmptyConfig() { + manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithEmptyConfig)) + d.Require().Nil(err) + manifestMediaType, content, err := manifest.Payload() + d.Require().Nil(err) + + art := &artifact.Artifact{ManifestMediaType: manifestMediaType, MediaType: "application/vnd.oci.empty.v1+json"} + err = d.processor.AbstractMetadata(context.TODO(), art, content) + d.Assert().Equal(0, len(art.ExtraAttrs)) + d.Assert().Equal(2, len(emptyConfig)) + d.Require().Nil(err) } func TestDefaultProcessorTestSuite(t *testing.T) { diff --git a/src/go.mod b/src/go.mod index 72cce223f..07ff72277 100644 --- a/src/go.mod +++ b/src/go.mod @@ -46,7 +46,7 @@ require ( github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/olekukonko/tablewriter v0.0.5 github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/image-spec v1.1.0-rc5 + github.com/opencontainers/image-spec v1.1.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.17.0 github.com/robfig/cron/v3 v3.0.1 diff --git a/src/go.sum b/src/go.sum index 2ca9fa57f..b8532ba34 100644 --- a/src/go.sum +++ b/src/go.sum @@ -514,8 +514,8 @@ github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= -github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= diff --git a/src/pkg/artifact/dao/model.go b/src/pkg/artifact/dao/model.go index c9ba1447b..7bb7c36ab 100644 --- a/src/pkg/artifact/dao/model.go +++ b/src/pkg/artifact/dao/model.go @@ -33,6 +33,7 @@ type Artifact struct { Type string `orm:"column(type)"` // image, chart or other OCI compatible MediaType string `orm:"column(media_type)"` // the media type of artifact ManifestMediaType string `orm:"column(manifest_media_type)"` // the media type of manifest/index + ArtifactType string `orm:"colume(artifact_type)"` // the artifactType of manifest/index ProjectID int64 `orm:"column(project_id)"` // needed for quota RepositoryID int64 `orm:"column(repository_id)"` RepositoryName string `orm:"column(repository_name)"` diff --git a/src/pkg/artifact/model.go b/src/pkg/artifact/model.go index 944e370b4..464a31924 100644 --- a/src/pkg/artifact/model.go +++ b/src/pkg/artifact/model.go @@ -34,6 +34,7 @@ type Artifact struct { Type string `json:"type"` // image, chart or other OCI compatible MediaType string `json:"media_type"` // the media type of artifact. Mostly, it's the value of `manifest.config.mediatype` ManifestMediaType string `json:"manifest_media_type"` // the media type of manifest/index + ArtifactType string `json:"artifact_type"` // the artifactType of manifest/index ProjectID int64 `json:"project_id"` RepositoryID int64 `json:"repository_id"` RepositoryName string `json:"repository_name"` @@ -63,6 +64,7 @@ func (a *Artifact) From(art *dao.Artifact) { a.Type = art.Type a.MediaType = art.MediaType a.ManifestMediaType = art.ManifestMediaType + a.ArtifactType = art.ArtifactType a.ProjectID = art.ProjectID a.RepositoryID = art.RepositoryID a.RepositoryName = art.RepositoryName @@ -92,6 +94,7 @@ func (a *Artifact) To() *dao.Artifact { Type: a.Type, MediaType: a.MediaType, ManifestMediaType: a.ManifestMediaType, + ArtifactType: a.ArtifactType, ProjectID: a.ProjectID, RepositoryID: a.RepositoryID, RepositoryName: a.RepositoryName, diff --git a/src/pkg/artifact/model_test.go b/src/pkg/artifact/model_test.go index 537658faf..24bd77eba 100644 --- a/src/pkg/artifact/model_test.go +++ b/src/pkg/artifact/model_test.go @@ -37,6 +37,7 @@ func (m *modelTestSuite) TestArtifactFrom() { Type: "IMAGE", MediaType: "application/vnd.oci.image.config.v1+json", ManifestMediaType: "application/vnd.oci.image.manifest.v1+json", + ArtifactType: "application/vnd.example+type", ProjectID: 1, RepositoryID: 1, Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180", @@ -52,6 +53,7 @@ func (m *modelTestSuite) TestArtifactFrom() { assert.Equal(t, dbArt.Type, art.Type) assert.Equal(t, dbArt.MediaType, art.MediaType) assert.Equal(t, dbArt.ManifestMediaType, art.ManifestMediaType) + assert.Equal(t, dbArt.ArtifactType, art.ArtifactType) assert.Equal(t, dbArt.ProjectID, art.ProjectID) assert.Equal(t, dbArt.RepositoryID, art.RepositoryID) assert.Equal(t, dbArt.Digest, art.Digest) @@ -71,6 +73,7 @@ func (m *modelTestSuite) TestArtifactTo() { RepositoryID: 1, MediaType: "application/vnd.oci.image.config.v1+json", ManifestMediaType: "application/vnd.oci.image.manifest.v1+json", + ArtifactType: "application/vnd.example+type", Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180", Size: 1024, PushTime: time.Now(), @@ -87,6 +90,7 @@ func (m *modelTestSuite) TestArtifactTo() { assert.Equal(t, art.Type, dbArt.Type) assert.Equal(t, art.MediaType, dbArt.MediaType) assert.Equal(t, art.ManifestMediaType, dbArt.ManifestMediaType) + assert.Equal(t, art.ArtifactType, dbArt.ArtifactType) assert.Equal(t, art.ProjectID, dbArt.ProjectID) assert.Equal(t, art.RepositoryID, dbArt.RepositoryID) assert.Equal(t, art.Digest, dbArt.Digest) diff --git a/src/pkg/blob/dao/dao_test.go b/src/pkg/blob/dao/dao_test.go index 00c63e94c..69771a89f 100644 --- a/src/pkg/blob/dao/dao_test.go +++ b/src/pkg/blob/dao/dao_test.go @@ -269,7 +269,7 @@ func (suite *DaoTestSuite) TestFindBlobsShouldUnassociatedWithProject() { artifact1 := suite.DigestString() artifact2 := suite.DigestString() - sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world')` + sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name, artifact_type) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world', 'artifact_type')` suite.ExecSQL(sql, artifact1, projectID, 10) suite.ExecSQL(sql, artifact2, projectID, 10) diff --git a/src/pkg/blob/manager_test.go b/src/pkg/blob/manager_test.go index c688bb34e..8ed541970 100644 --- a/src/pkg/blob/manager_test.go +++ b/src/pkg/blob/manager_test.go @@ -130,7 +130,7 @@ func (suite *ManagerTestSuite) TestCleanupAssociationsForProject() { artifact1 := suite.DigestString() artifact2 := suite.DigestString() - sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world')` + sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name, artifact_type) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world', 'artifact_type')` suite.ExecSQL(sql, artifact1, projectID, 10) suite.ExecSQL(sql, artifact2, projectID, 10) @@ -200,7 +200,7 @@ func (suite *ManagerTestSuite) TestFindBlobsShouldUnassociatedWithProject() { artifact1 := suite.DigestString() artifact2 := suite.DigestString() - sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world')` + sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name, artifact_type) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world', 'artifact_type')` suite.ExecSQL(sql, artifact1, projectID, 11) suite.ExecSQL(sql, artifact2, projectID, 11) diff --git a/src/pkg/securityhub/dao/security_test.go b/src/pkg/securityhub/dao/security_test.go index f6033b75f..ae6025b30 100644 --- a/src/pkg/securityhub/dao/security_test.go +++ b/src/pkg/securityhub/dao/security_test.go @@ -49,12 +49,12 @@ func (suite *SecurityDaoTestSuite) SetupTest() { `delete from artifact_accessory`, `delete from artifact`, `insert into scan_report(uuid, digest, registration_uuid, mime_type, critical_cnt, high_cnt, medium_cnt, low_cnt, unknown_cnt, fixable_cnt) values('uuid', 'digest1001', 'ruuid', 'application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0', 50, 50, 50, 0, 0, 20)`, - `insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon) -values (1001, 1, 'library/hello-world', 'digest1001', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.docker.distribution.manifest.v2+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '');`, - `insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon) -values (1002, 1, 'library/hello-world', 'digest1002', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.oci.image.config.v1+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '');`, - `insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon) -values (1003, 1, 'library/hello-world', 'digest1003', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.oci.image.config.v1+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '');`, + `insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon, artifact_type) +values (1001, 1, 'library/hello-world', 'digest1001', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.docker.distribution.manifest.v2+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '', 'application/vnd.docker.container.image.v1+json');`, + `insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon, artifact_type) +values (1002, 1, 'library/hello-world', 'digest1002', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.oci.image.config.v1+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '', 'application/vnd.docker.container.image.v1+json');`, + `insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon, artifact_type) +values (1003, 1, 'library/hello-world', 'digest1003', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.oci.image.config.v1+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '', 'application/vnd.docker.container.image.v1+json');`, `insert into tag (id, repository_id, artifact_id, name, push_time, pull_time) values (1001, 1742, 1001, 'latest', '2023-06-02 01:45:55.050785', '2023-06-02 09:16:47.838778')`, `INSERT INTO artifact_accessory (id, artifact_id, subject_artifact_id, type, size, digest, creation_time, subject_artifact_digest, subject_artifact_repo) VALUES (1001, 1002, 1, 'signature.cosign', 2109, 'sha256:08c64c0de2667abcf3974b4b75b82903f294680b81584318adc4826d0dcb7a9c', '2023-08-03 04:54:32.102928', 'sha256:a97a153152fcd6410bdf4fb64f5622ecf97a753f07dcc89dab14509d059736cf', 'library/nuxeo')`, `INSERT INTO artifact_reference (id, parent_id, child_id, child_digest, platform, urls, annotations) VALUES (1001, 1001, 1003, 'sha256:d2b2f2980e9ccc570e5726b56b54580f23a018b7b7314c9eaff7e5e479c78657', '{"architecture":"amd64","os":"linux"}', '', null)`, diff --git a/src/server/middleware/subject/subject.go b/src/server/middleware/subject/subject.go index 0719b5c34..c4b86863e 100644 --- a/src/server/middleware/subject/subject.go +++ b/src/server/middleware/subject/subject.go @@ -106,6 +106,15 @@ func Middleware() func(http.Handler) http.Handler { return err } + /* + when an images is pushed, it could be + 1. single image (do nothing) + 2. an accesory image + 3. a subject image + 4. both as an accessory and a subject image + and a subject image or accessory image could be pushed in either order + */ + if mf.Subject != nil { subjectArt, err := artifact.Ctl.GetByReference(ctx, info.Repository, mf.Subject.Digest.String(), nil) if err != nil { @@ -113,7 +122,7 @@ func Middleware() func(http.Handler) http.Handler { logger.Errorf("failed to get subject artifact: %s, error: %v", mf.Subject.Digest, err) return err } - log.Debug("the subject of the signature doesn't exist.") + log.Debug("the subject artifact doesn't exist.") } art, err := artifact.Ctl.GetByReference(ctx, info.Repository, info.Reference, nil) if err != nil { @@ -128,7 +137,12 @@ func Middleware() func(http.Handler) http.Handler { Digest: art.Digest, } accData.Type = model.TypeSubject - switch mf.Config.MediaType { + // since oci-spec 1.1, image type may from artifactType if presents, otherwise would be Config.MediaType + fromType := mf.Config.MediaType + if mf.ArtifactType != "" { + fromType = mf.ArtifactType + } + switch fromType { case ocispec.MediaTypeImageConfig, schema2.MediaTypeImageConfig: if isNydusImage(mf) { accData.Type = model.TypeNydusAccelerator @@ -152,18 +166,18 @@ func Middleware() func(http.Handler) http.Handler { // when subject artifact is pushed after accessory artifact, current subject artifact do not exist. // so we use reference manifest subject digest instead of subjectArt.Digest w.Header().Set("OCI-Subject", mf.Subject.Digest.String()) - } else { + } + + // check if images is a Subject artifact + digest := digest.FromBytes(body) + accs, err := accessory.Mgr.List(ctx, q.New(q.KeyWords{"SubjectArtifactDigest": digest, "SubjectArtifactRepo": info.Repository})) + if err != nil { + logger.Errorf("failed to list accessory artifact: %s, error: %v", digest, err) + return err + } + if len(accs) > 0 { // In certain cases, the OCI client may push the subject artifact and accessory in either order. // Therefore, it is necessary to handle situations where the client pushes the accessory ahead of the subject artifact. - digest := digest.FromBytes(body) - accs, err := accessory.Mgr.List(ctx, q.New(q.KeyWords{"SubjectArtifactDigest": digest, "SubjectArtifactRepo": info.Repository})) - if err != nil { - logger.Errorf("failed to list accessory artifact: %s, error: %v", digest, err) - return err - } - if len(accs) <= 0 { - return nil - } art, err := artifact.Ctl.GetByReference(ctx, info.Repository, digest.String(), nil) if err != nil { logger.Errorf("failed to list artifact: %s, error: %v", digest, err) diff --git a/src/server/registry/referrers.go b/src/server/registry/referrers.go index 799d04acd..ee715faba 100644 --- a/src/server/registry/referrers.go +++ b/src/server/registry/referrers.go @@ -89,11 +89,11 @@ func (r *referrersHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { Size: accArt.Size, Digest: digest.Digest(accArt.Digest), Annotations: accArt.Annotations, - ArtifactType: accArt.MediaType, + ArtifactType: accArt.ArtifactType, } - // filter by the artifactType since the artifactType is actually the config media type of the artifact. + // filter use accArt.ArtifactType as artifactType if at != "" { - if accArt.MediaType == at { + if accArt.ArtifactType == at { mfs = append(mfs, mf) } } else { diff --git a/src/server/registry/referrers_test.go b/src/server/registry/referrers_test.go index 27f1deec3..f8d8abc22 100644 --- a/src/server/registry/referrers_test.go +++ b/src/server/registry/referrers_test.go @@ -37,7 +37,8 @@ func TestReferrersHandlerOK(t *testing.T) { Return(&artifact.Artifact{ Digest: digestVal, ManifestMediaType: "application/vnd.oci.image.manifest.v1+json", - MediaType: "application/vnd.example.main", + MediaType: "application/vnd.example.sbom", + ArtifactType: "application/vnd.example+type", Size: 1000, Annotations: map[string]string{ "name": "test-image", @@ -72,8 +73,8 @@ func TestReferrersHandlerOK(t *testing.T) { } index := &ocispec.Index{} json.Unmarshal([]byte(rec.Body.String()), index) - if index.Manifests[0].ArtifactType != "application/vnd.example.main" { - t.Errorf("Expected response body %s, but got %s", "application/vnd.example.main", rec.Body.String()) + if index.Manifests[0].ArtifactType != "application/vnd.example+type" { + t.Errorf("Expected response body %s, but got %s", "application/vnd.example+type", rec.Body.String()) } }