Upstream
/
harbor
mirror of https://github.com/goharbor/harbor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Update support for artifactType for both manifest and index (#20030) 

add artifact_type for artifact model to support artifactType filter

Signed-off-by: yminer <yminer@vmware.com>

add 2.11 sql schema & update index artifactType omitted

Signed-off-by: yminer <yminer@vmware.com>

update UT

update migrate sql for artifact_type

Signed-off-by: yminer <yminer@vmware.com>

remove debug line
This commit is contained in:
MinerYang 2024-03-12 21:52:56 +08:00 committed by GitHub
parent dbe9790147
commit a269b4f31c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 374 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
make/migrations/postgresql/0140_2.11.0_schema.up.sql Normal file
View File

@ -0,0 +1,31 @@
/*
table artifact:
id SERIAL PRIMARY KEY NOT NULL,
type varchar(255) NOT NULL,
media_type varchar(255) NOT NULL,
manifest_media_type varchar(255) NOT NULL,
artifact_type varchar(255) NOT NULL,
project_id int NOT NULL,
repository_id int NOT NULL,
repository_name varchar(255) NOT NULL,
digest varchar(255) NOT NULL,
size bigint,
push_time timestamp default CURRENT_TIMESTAMP,
pull_time timestamp,
extra_attrs text,
annotations jsonb,
CONSTRAINT unique_artifact UNIQUE (repository_id, digest)
*/
/*
Add new column artifact_type for artifact table to work with oci-spec v1.1.0 list referrer api
*/
ALTER TABLE artifact ADD COLUMN artifact_type varchar(255);
/*
set value for artifact_type
then set column artifact_type as not null
*/
UPDATE artifact SET artifact_type = media_type;
ALTER TABLE artifact ALTER COLUMN artifact_type SET NOT NULL;

20
src/controller/artifact/abstractor.go
View File

@ -127,10 +127,18 @@ func (a *abstractor) abstractManifestV2Metadata(artifact *artifact.Artifact, con
}
// use the "manifest.config.mediatype" as the media type of the artifact
artifact.MediaType = manifest.Config.MediaType
if manifest.Annotations[wasm.AnnotationVariantKey] == wasm.AnnotationVariantValue || manifest.Annotations[wasm.AnnotationHandlerKey] == wasm.AnnotationHandlerValue {
artifact.MediaType = wasm.MediaType
}
/*
https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#listing-referrers
For referrers list, if the artifactType is empty or missing in the image manifest, the value of artifactType MUST be set to the config descriptor mediaType value
*/
if manifest.ArtifactType != "" {
artifact.ArtifactType = manifest.ArtifactType
} else {
artifact.ArtifactType = manifest.Config.MediaType
}
// set size
artifact.Size = int64(len(content)) + manifest.Config.Size
@ -153,6 +161,16 @@ func (a *abstractor) abstractIndexMetadata(ctx context.Context, art *artifact.Ar
return err
}
/*
https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#listing-referrers
For referrers list, If the artifactType is empty or missing in an index, the artifactType MUST be omitted.
*/
if index.ArtifactType != "" {
art.ArtifactType = index.ArtifactType
} else {
art.ArtifactType = ""
}
// set annotations
art.Annotations = index.Annotations

177
src/controller/artifact/abstractor_test.go
View File

@ -15,6 +15,7 @@
package artifact
import (
"context"
"testing"
"github.com/docker/distribution"
@ -175,7 +176,66 @@ var (
"com.example.key1": "value1"
}
}`
OCIManifest = `{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"config": {
"mediaType": "application/vnd.example.config.v1+json",
"digest": "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03",
"size": 123
},
"layers": [
{
"mediaType": "application/vnd.example.data.v1.tar+gzip",
"digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317",
"size": 1234
}
],
"annotations": {
"com.example.key1": "value1"
}
}`
OCIManifestWithArtifactType = `{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"artifactType": "application/vnd.example+type",
"config": {
"mediaType": "application/vnd.example.config.v1+json",
"digest": "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03",
"size": 123
},
"layers": [
{
"mediaType": "application/vnd.example.data.v1.tar+gzip",
"digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317",
"size": 1234
}
],
"annotations": {
"com.example.key1": "value1"
}
}`
OCIManifestWithEmptyConfig = `{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"artifactType": "application/vnd.example+type",
"config": {
"mediaType": "application/vnd.oci.empty.v1+json",
"digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
"size": 2
},
"layers": [
{
"mediaType": "application/vnd.example+type",
"digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317",
"size": 1234
}
],
"annotations": {
"oci.opencontainers.image.created": "2023-01-02T03:04:05Z",
"com.example.data": "payload"
}
}`
index = `{
"schemaVersion": 2,
"manifests": [
@ -202,6 +262,34 @@ var (
"com.example.key1": "value1"
}
}`
indexWithArtifactType = `{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.index.v1+json",
"artifactType": "application/vnd.food.stand",
"manifests": [
{
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"size": 7143,
"digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
"platform": {
"architecture": "ppc64le",
"os": "linux"
}
},
{
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"size": 7682,
"digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
"platform": {
"architecture": "amd64",
"os": "linux"
}
}
],
"annotations": {
"com.example.key1": "value1"
}
}`
)
type abstractorTestSuite struct {
@ -267,6 +355,67 @@ func (a *abstractorTestSuite) TestAbstractMetadataOfV2Manifest() {
a.Equal("value1", artifact.Annotations["com.example.key1"])
}
// oci-spec v1
func (a *abstractorTestSuite) TestAbstractMetadataOfOCIManifest() {
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifest))
a.Require().Nil(err)
a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil)
artifact := &artifact.Artifact{
ID: 1,
}
a.processor.On("AbstractMetadata", mock.Anything, mock.Anything, mock.Anything).Return(nil)
err = a.abstractor.AbstractMetadata(context.TODO(), artifact)
a.Require().Nil(err)
a.Assert().Equal(int64(1), artifact.ID)
a.Assert().Equal(v1.MediaTypeImageManifest, artifact.ManifestMediaType)
a.Assert().Equal("application/vnd.example.config.v1+json", artifact.MediaType)
a.Assert().Equal("application/vnd.example.config.v1+json", artifact.ArtifactType)
a.Assert().Equal(int64(1916), artifact.Size)
a.Require().Len(artifact.Annotations, 1)
a.Equal("value1", artifact.Annotations["com.example.key1"])
}
// oci-spec v1.1.0 with artifactType
func (a *abstractorTestSuite) TestAbstractMetadataOfOCIManifestWithArtifactType() {
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithArtifactType))
a.Require().Nil(err)
a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil)
artifact := &artifact.Artifact{
ID: 1,
}
a.processor.On("AbstractMetadata", mock.Anything, mock.Anything, mock.Anything).Return(nil)
err = a.abstractor.AbstractMetadata(context.TODO(), artifact)
a.Require().Nil(err)
a.Assert().Equal(int64(1), artifact.ID)
a.Assert().Equal(v1.MediaTypeImageManifest, artifact.ManifestMediaType)
a.Assert().Equal("application/vnd.example.config.v1+json", artifact.MediaType)
a.Assert().Equal("application/vnd.example+type", artifact.ArtifactType)
a.Assert().Equal(int64(1966), artifact.Size)
a.Require().Len(artifact.Annotations, 1)
a.Equal("value1", artifact.Annotations["com.example.key1"])
}
// empty config with artifactType
func (a *abstractorTestSuite) TestAbstractMetadataOfV2ManifestWithEmptyConfig() {
// v1.MediaTypeImageManifest
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithEmptyConfig))
a.Require().Nil(err)
a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil)
artifact := &artifact.Artifact{
ID: 1,
}
a.processor.On("AbstractMetadata", mock.Anything, mock.Anything, mock.Anything).Return(nil)
err = a.abstractor.AbstractMetadata(context.TODO(), artifact)
a.Require().Nil(err)
a.Assert().Equal(int64(1), artifact.ID)
a.Assert().Equal(v1.MediaTypeImageManifest, artifact.ManifestMediaType)
a.Assert().Equal(v1.MediaTypeEmptyJSON, artifact.MediaType)
a.Assert().Equal("application/vnd.example+type", artifact.ArtifactType)
a.Assert().Equal(int64(1880), artifact.Size)
a.Require().Len(artifact.Annotations, 2)
a.Equal("payload", artifact.Annotations["com.example.data"])
}
// OCI index
func (a *abstractorTestSuite) TestAbstractMetadataOfIndex() {
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageIndex, []byte(index))
@ -279,17 +428,41 @@ func (a *abstractorTestSuite) TestAbstractMetadataOfIndex() {
artifact := &artifact.Artifact{
ID: 1,
}
err = a.abstractor.AbstractMetadata(nil, artifact)
err = a.abstractor.AbstractMetadata(context.TODO(), artifact)
a.Require().Nil(err)
a.Assert().Equal(int64(1), artifact.ID)
a.Assert().Equal(v1.MediaTypeImageIndex, artifact.ManifestMediaType)
a.Assert().Equal(v1.MediaTypeImageIndex, artifact.MediaType)
a.Assert().Equal("", artifact.ArtifactType)
a.Assert().Equal(int64(668), artifact.Size)
a.Require().Len(artifact.Annotations, 1)
a.Assert().Equal("value1", artifact.Annotations["com.example.key1"])
a.Len(artifact.References, 2)
}
func (a *abstractorTestSuite) TestAbstractMetadataOfIndexWithArtifactType() {
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageIndex, []byte(indexWithArtifactType))
a.Require().Nil(err)
a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil)
a.argMgr.On("GetByDigest", mock.Anything, mock.Anything, mock.Anything).Return(&artifact.Artifact{
ID: 2,
Size: 10,
}, nil)
artifact := &artifact.Artifact{
ID: 1,
}
err = a.abstractor.AbstractMetadata(context.TODO(), artifact)
a.Require().Nil(err)
a.Assert().Equal(int64(1), artifact.ID)
a.Assert().Equal(v1.MediaTypeImageIndex, artifact.ManifestMediaType)
a.Assert().Equal(v1.MediaTypeImageIndex, artifact.MediaType)
a.Assert().Equal("application/vnd.food.stand", artifact.ArtifactType)
a.Assert().Equal(int64(801), artifact.Size)
a.Require().Len(artifact.Annotations, 1)
a.Assert().Equal("value1", artifact.Annotations["com.example.key1"])
a.Len(artifact.References, 2)
}
type unknownManifest struct{}
func (u *unknownManifest) References() []distribution.Descriptor {

105
src/controller/artifact/processor/default_test.go
View File

@ -117,7 +117,31 @@ var (
}
]
}`
v2ManifestWithUnknownConfig = `{
OCIManifestWithUnknownJsonConfig = `{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"config": {
"mediaType": "application/vnd.exmaple.config.v1+json",
"digest": "sha256:48ef4a53c0770222d9752cd0588431dbda54667046208c79804e34c15c1579cd",
"size": 129
},
"layers": [
{
"mediaType": "application/vnd.example.data.v1.tar+gzip",
"digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317",
"size": 1234
}
],
"annotations": {
"com.example.key1": "value1"
}
}`
UnknownJsonConfig = `{
"author": "yminer",
"architecture": "amd64",
"selfdefined": "true"
}`
OCIManifestWithUnknownConfig = `{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"config": {
@ -141,7 +165,30 @@ var (
"newUnspecifiedField": null
}
}`
unknownConfig = `{NHL Peanut Butter on my NHL bagel}`
UnknownConfig = `{NHL Peanut Butter on my NHL bagel}`
OCIManifestWithEmptyConfig = `{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"artifactType": "application/vnd.example+type",
"config": {
"mediaType": "application/vnd.oci.empty.v1+json",
"digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
"size": 2
},
"layers": [
{
"mediaType": "application/vnd.example+type",
"digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317",
"size": 1234
}
],
"annotations": {
"oci.opencontainers.image.created": "2023-01-02T03:04:05Z",
"com.example.data": "payload"
}
}`
emptyConfig = `{}`
)
type defaultProcessorTestSuite struct {
@ -190,6 +237,12 @@ func (d *defaultProcessorTestSuite) TestGetArtifactType() {
typee = processor.GetArtifactType(nil, art)
d.Equal("IMAGE", typee)
mediaType = "application/vnd.example.config.v1+json"
art = &artifact.Artifact{MediaType: mediaType}
processor = &defaultProcessor{}
typee = processor.GetArtifactType(nil, art)
d.Equal(ArtifactTypeUnknown, typee)
mediaType = "application/vnd.cncf.helm.chart.config.v1+json"
art = &artifact.Artifact{MediaType: mediaType}
processor = &defaultProcessor{}
@ -229,19 +282,53 @@ func (d *defaultProcessorTestSuite) TestAbstractMetadata() {
d.Len(art.ExtraAttrs, 12)
}
func (d *defaultProcessorTestSuite) TestAbstractMetadataWithUnknownConfig() {
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(v2ManifestWithUnknownConfig))
func (d *defaultProcessorTestSuite) TestAbstractMetadataOfOCIManifesttWithUnknownJsonConfig() {
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithUnknownJsonConfig))
d.Require().Nil(err)
manifestMediaType, content, err := manifest.Payload()
d.Require().Nil(err)
configBlob := io.NopCloser(strings.NewReader(unknownConfig))
d.regCli.On("PullBlob", mock.Anything, mock.Anything).Return(int64(0), configBlob, nil)
art := &artifact.Artifact{ManifestMediaType: manifestMediaType}
err = d.processor.AbstractMetadata(nil, art, content)
configBlob := io.NopCloser(strings.NewReader(UnknownJsonConfig))
metadata := map[string]interface{}{}
err = json.NewDecoder(configBlob).Decode(&metadata)
d.Require().Nil(err)
art := &artifact.Artifact{ManifestMediaType: manifestMediaType, MediaType: "application/vnd.example.config.v1+json"}
d.regCli.On("PullBlob", mock.Anything, mock.Anything).Return(int64(129), configBlob, nil)
d.parser.On("Parse", context.TODO(), mock.AnythingOfType("*artifact.Artifact"), mock.AnythingOfType("[]byte")).Return(nil)
err = d.processor.AbstractMetadata(context.TODO(), art, content)
d.Require().Nil(err)
d.Len(art.ExtraAttrs, 0)
d.Len(unknownConfig, 35)
d.NotEqual(art.ExtraAttrs, len(metadata))
}
func (d *defaultProcessorTestSuite) TestAbstractMetadataWithUnknownConfig() {
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithUnknownConfig))
d.Require().Nil(err)
manifestMediaType, content, err := manifest.Payload()
d.Require().Nil(err)
configBlob := io.NopCloser(strings.NewReader(UnknownConfig))
d.regCli.On("PullBlob", mock.Anything, mock.Anything).Return(int64(0), configBlob, nil)
art := &artifact.Artifact{ManifestMediaType: manifestMediaType, MediaType: "application/vnd.nhl.peanut.butter.bagel"}
err = d.processor.AbstractMetadata(context.TODO(), art, content)
d.Require().Nil(err)
d.Len(art.ExtraAttrs, 0)
}
func (d *defaultProcessorTestSuite) TestAbstractMetadataWithEmptyConfig() {
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithEmptyConfig))
d.Require().Nil(err)
manifestMediaType, content, err := manifest.Payload()
d.Require().Nil(err)
art := &artifact.Artifact{ManifestMediaType: manifestMediaType, MediaType: "application/vnd.oci.empty.v1+json"}
err = d.processor.AbstractMetadata(context.TODO(), art, content)
d.Assert().Equal(0, len(art.ExtraAttrs))
d.Assert().Equal(2, len(emptyConfig))
d.Require().Nil(err)
}
func TestDefaultProcessorTestSuite(t *testing.T) {

2
src/go.mod
View File

@ -46,7 +46,7 @@ require (
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
github.com/olekukonko/tablewriter v0.0.5
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.0-rc5
github.com/opencontainers/image-spec v1.1.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.17.0
github.com/robfig/cron/v3 v3.0.1

4
src/go.sum
View File

@ -514,8 +514,8 @@ github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI=
github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=

1
src/pkg/artifact/dao/model.go
View File

@ -33,6 +33,7 @@ type Artifact struct {
Type string `orm:"column(type)"` // image, chart or other OCI compatible
MediaType string `orm:"column(media_type)"` // the media type of artifact
ManifestMediaType string `orm:"column(manifest_media_type)"` // the media type of manifest/index
ArtifactType string `orm:"colume(artifact_type)"` // the artifactType of manifest/index
ProjectID int64 `orm:"column(project_id)"` // needed for quota
RepositoryID int64 `orm:"column(repository_id)"`
RepositoryName string `orm:"column(repository_name)"`

3
src/pkg/artifact/model.go
View File

@ -34,6 +34,7 @@ type Artifact struct {
Type string `json:"type"` // image, chart or other OCI compatible
MediaType string `json:"media_type"` // the media type of artifact. Mostly, it's the value of `manifest.config.mediatype`
ManifestMediaType string `json:"manifest_media_type"` // the media type of manifest/index
ArtifactType string `json:"artifact_type"` // the artifactType of manifest/index
ProjectID int64 `json:"project_id"`
RepositoryID int64 `json:"repository_id"`
RepositoryName string `json:"repository_name"`
@ -63,6 +64,7 @@ func (a *Artifact) From(art *dao.Artifact) {
a.Type = art.Type
a.MediaType = art.MediaType
a.ManifestMediaType = art.ManifestMediaType
a.ArtifactType = art.ArtifactType
a.ProjectID = art.ProjectID
a.RepositoryID = art.RepositoryID
a.RepositoryName = art.RepositoryName
@ -92,6 +94,7 @@ func (a *Artifact) To() *dao.Artifact {
Type: a.Type,
MediaType: a.MediaType,
ManifestMediaType: a.ManifestMediaType,
ArtifactType: a.ArtifactType,
ProjectID: a.ProjectID,
RepositoryID: a.RepositoryID,
RepositoryName: a.RepositoryName,

4
src/pkg/artifact/model_test.go
View File

@ -37,6 +37,7 @@ func (m *modelTestSuite) TestArtifactFrom() {
Type: "IMAGE",
MediaType: "application/vnd.oci.image.config.v1+json",
ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
ArtifactType: "application/vnd.example+type",
ProjectID: 1,
RepositoryID: 1,
Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
@ -52,6 +53,7 @@ func (m *modelTestSuite) TestArtifactFrom() {
assert.Equal(t, dbArt.Type, art.Type)
assert.Equal(t, dbArt.MediaType, art.MediaType)
assert.Equal(t, dbArt.ManifestMediaType, art.ManifestMediaType)
assert.Equal(t, dbArt.ArtifactType, art.ArtifactType)
assert.Equal(t, dbArt.ProjectID, art.ProjectID)
assert.Equal(t, dbArt.RepositoryID, art.RepositoryID)
assert.Equal(t, dbArt.Digest, art.Digest)
@ -71,6 +73,7 @@ func (m *modelTestSuite) TestArtifactTo() {
RepositoryID: 1,
MediaType: "application/vnd.oci.image.config.v1+json",
ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
ArtifactType: "application/vnd.example+type",
Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
Size: 1024,
PushTime: time.Now(),
@ -87,6 +90,7 @@ func (m *modelTestSuite) TestArtifactTo() {
assert.Equal(t, art.Type, dbArt.Type)
assert.Equal(t, art.MediaType, dbArt.MediaType)
assert.Equal(t, art.ManifestMediaType, dbArt.ManifestMediaType)
assert.Equal(t, art.ArtifactType, dbArt.ArtifactType)
assert.Equal(t, art.ProjectID, dbArt.ProjectID)
assert.Equal(t, art.RepositoryID, dbArt.RepositoryID)
assert.Equal(t, art.Digest, dbArt.Digest)

2
src/pkg/blob/dao/dao_test.go
View File

@ -269,7 +269,7 @@ func (suite *DaoTestSuite) TestFindBlobsShouldUnassociatedWithProject() {
artifact1 := suite.DigestString()
artifact2 := suite.DigestString()
sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world')`
sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name, artifact_type) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world', 'artifact_type')`
suite.ExecSQL(sql, artifact1, projectID, 10)
suite.ExecSQL(sql, artifact2, projectID, 10)

4
src/pkg/blob/manager_test.go
View File

@ -130,7 +130,7 @@ func (suite *ManagerTestSuite) TestCleanupAssociationsForProject() {
artifact1 := suite.DigestString()
artifact2 := suite.DigestString()
sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world')`
sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name, artifact_type) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world', 'artifact_type')`
suite.ExecSQL(sql, artifact1, projectID, 10)
suite.ExecSQL(sql, artifact2, projectID, 10)
@ -200,7 +200,7 @@ func (suite *ManagerTestSuite) TestFindBlobsShouldUnassociatedWithProject() {
artifact1 := suite.DigestString()
artifact2 := suite.DigestString()
sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world')`
sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name, artifact_type) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world', 'artifact_type')`
suite.ExecSQL(sql, artifact1, projectID, 11)
suite.ExecSQL(sql, artifact2, projectID, 11)

12
src/pkg/securityhub/dao/security_test.go
View File

@ -49,12 +49,12 @@ func (suite *SecurityDaoTestSuite) SetupTest() {
`delete from artifact_accessory`,
`delete from artifact`,
`insert into scan_report(uuid, digest, registration_uuid, mime_type, critical_cnt, high_cnt, medium_cnt, low_cnt, unknown_cnt, fixable_cnt) values('uuid', 'digest1001', 'ruuid', 'application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0', 50, 50, 50, 0, 0, 20)`,
`insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon)
values (1001, 1, 'library/hello-world', 'digest1001', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.docker.distribution.manifest.v2+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '');`,
`insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon)
values (1002, 1, 'library/hello-world', 'digest1002', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.oci.image.config.v1+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '');`,
`insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon)
values (1003, 1, 'library/hello-world', 'digest1003', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.oci.image.config.v1+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '');`,
`insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon, artifact_type)
values (1001, 1, 'library/hello-world', 'digest1001', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.docker.distribution.manifest.v2+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '', 'application/vnd.docker.container.image.v1+json');`,
`insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon, artifact_type)
values (1002, 1, 'library/hello-world', 'digest1002', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.oci.image.config.v1+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '', 'application/vnd.docker.container.image.v1+json');`,
`insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon, artifact_type)
values (1003, 1, 'library/hello-world', 'digest1003', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.oci.image.config.v1+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '', 'application/vnd.docker.container.image.v1+json');`,
`insert into tag (id, repository_id, artifact_id, name, push_time, pull_time) values (1001, 1742, 1001, 'latest', '2023-06-02 01:45:55.050785', '2023-06-02 09:16:47.838778')`,
`INSERT INTO artifact_accessory (id, artifact_id, subject_artifact_id, type, size, digest, creation_time, subject_artifact_digest, subject_artifact_repo) VALUES (1001, 1002, 1, 'signature.cosign', 2109, 'sha256:08c64c0de2667abcf3974b4b75b82903f294680b81584318adc4826d0dcb7a9c', '2023-08-03 04:54:32.102928', 'sha256:a97a153152fcd6410bdf4fb64f5622ecf97a753f07dcc89dab14509d059736cf', 'library/nuxeo')`,
`INSERT INTO artifact_reference (id, parent_id, child_id, child_digest, platform, urls, annotations) VALUES (1001, 1001, 1003, 'sha256:d2b2f2980e9ccc570e5726b56b54580f23a018b7b7314c9eaff7e5e479c78657', '{"architecture":"amd64","os":"linux"}', '', null)`,

38
src/server/middleware/subject/subject.go
View File

@ -106,6 +106,15 @@ func Middleware() func(http.Handler) http.Handler {
return err
}
/*
when an images is pushed, it could be
1. single image (do nothing)
2. an accesory image
3. a subject image
4. both as an accessory and a subject image
and a subject image or accessory image could be pushed in either order
*/
if mf.Subject != nil {
subjectArt, err := artifact.Ctl.GetByReference(ctx, info.Repository, mf.Subject.Digest.String(), nil)
if err != nil {
@ -113,7 +122,7 @@ func Middleware() func(http.Handler) http.Handler {
logger.Errorf("failed to get subject artifact: %s, error: %v", mf.Subject.Digest, err)
return err
}
log.Debug("the subject of the signature doesn't exist.")
log.Debug("the subject artifact doesn't exist.")
}
art, err := artifact.Ctl.GetByReference(ctx, info.Repository, info.Reference, nil)
if err != nil {
@ -128,7 +137,12 @@ func Middleware() func(http.Handler) http.Handler {
Digest: art.Digest,
}
accData.Type = model.TypeSubject
switch mf.Config.MediaType {
// since oci-spec 1.1, image type may from artifactType if presents, otherwise would be Config.MediaType
fromType := mf.Config.MediaType
if mf.ArtifactType != "" {
fromType = mf.ArtifactType
}
switch fromType {
case ocispec.MediaTypeImageConfig, schema2.MediaTypeImageConfig:
if isNydusImage(mf) {
accData.Type = model.TypeNydusAccelerator
@ -152,18 +166,18 @@ func Middleware() func(http.Handler) http.Handler {
// when subject artifact is pushed after accessory artifact, current subject artifact do not exist.
// so we use reference manifest subject digest instead of subjectArt.Digest
w.Header().Set("OCI-Subject", mf.Subject.Digest.String())
} else {
}
// check if images is a Subject artifact
digest := digest.FromBytes(body)
accs, err := accessory.Mgr.List(ctx, q.New(q.KeyWords{"SubjectArtifactDigest": digest, "SubjectArtifactRepo": info.Repository}))
if err != nil {
logger.Errorf("failed to list accessory artifact: %s, error: %v", digest, err)
return err
}
if len(accs) > 0 {
// In certain cases, the OCI client may push the subject artifact and accessory in either order.
// Therefore, it is necessary to handle situations where the client pushes the accessory ahead of the subject artifact.
digest := digest.FromBytes(body)
accs, err := accessory.Mgr.List(ctx, q.New(q.KeyWords{"SubjectArtifactDigest": digest, "SubjectArtifactRepo": info.Repository}))
if err != nil {
logger.Errorf("failed to list accessory artifact: %s, error: %v", digest, err)
return err
}
if len(accs) <= 0 {
return nil
}
art, err := artifact.Ctl.GetByReference(ctx, info.Repository, digest.String(), nil)
if err != nil {
logger.Errorf("failed to list artifact: %s, error: %v", digest, err)

6
src/server/registry/referrers.go
View File

@ -89,11 +89,11 @@ func (r *referrersHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
Size: accArt.Size,
Digest: digest.Digest(accArt.Digest),
Annotations: accArt.Annotations,
ArtifactType: accArt.MediaType,
ArtifactType: accArt.ArtifactType,
}
// filter by the artifactType since the artifactType is actually the config media type of the artifact.
// filter use accArt.ArtifactType as artifactType
if at != "" {
if accArt.MediaType == at {
if accArt.ArtifactType == at {
mfs = append(mfs, mf)
}
} else {

7
src/server/registry/referrers_test.go
View File

@ -37,7 +37,8 @@ func TestReferrersHandlerOK(t *testing.T) {
Return(&artifact.Artifact{
Digest: digestVal,
ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
MediaType: "application/vnd.example.main",
MediaType: "application/vnd.example.sbom",
ArtifactType: "application/vnd.example+type",
Size: 1000,
Annotations: map[string]string{
"name": "test-image",
@ -72,8 +73,8 @@ func TestReferrersHandlerOK(t *testing.T) {
}
index := &ocispec.Index{}
json.Unmarshal([]byte(rec.Body.String()), index)
if index.Manifests[0].ArtifactType != "application/vnd.example.main" {
t.Errorf("Expected response body %s, but got %s", "application/vnd.example.main", rec.Body.String())
if index.Manifests[0].ArtifactType != "application/vnd.example+type" {
t.Errorf("Expected response body %s, but got %s", "application/vnd.example+type", rec.Body.String())
}
}