mirror of
https://github.com/goharbor/harbor.git
synced 2024-11-22 02:05:41 +01:00
Update support for artifactType for both manifest and index (#20030)
add artifact_type for artifact model to support artifactType filter Signed-off-by: yminer <yminer@vmware.com> add 2.11 sql schema & update index artifactType omitted Signed-off-by: yminer <yminer@vmware.com> update UT update migrate sql for artifact_type Signed-off-by: yminer <yminer@vmware.com> remove debug line
This commit is contained in:
MinerYang
GitHub
parent
dbe9790147
commit
a269b4f31c
31
make/migrations/postgresql/0140_2.11.0_schema.up.sql
Normal file
31
make/migrations/postgresql/0140_2.11.0_schema.up.sql
Normal file
@ -0,0 +1,31 @@
|
||||
/*
|
||||
table artifact:
|
||||
id SERIAL PRIMARY KEY NOT NULL,
|
||||
type varchar(255) NOT NULL,
|
||||
media_type varchar(255) NOT NULL,
|
||||
manifest_media_type varchar(255) NOT NULL,
|
||||
artifact_type varchar(255) NOT NULL,
|
||||
project_id int NOT NULL,
|
||||
repository_id int NOT NULL,
|
||||
repository_name varchar(255) NOT NULL,
|
||||
digest varchar(255) NOT NULL,
|
||||
size bigint,
|
||||
push_time timestamp default CURRENT_TIMESTAMP,
|
||||
pull_time timestamp,
|
||||
extra_attrs text,
|
||||
annotations jsonb,
|
||||
CONSTRAINT unique_artifact UNIQUE (repository_id, digest)
|
||||
*/
|
||||
|
||||
/*
|
||||
Add new column artifact_type for artifact table to work with oci-spec v1.1.0 list referrer api
|
||||
*/
|
||||
ALTER TABLE artifact ADD COLUMN artifact_type varchar(255);
|
||||
|
||||
/*
|
||||
set value for artifact_type
|
||||
then set column artifact_type as not null
|
||||
*/
|
||||
UPDATE artifact SET artifact_type = media_type;
|
||||
|
||||
ALTER TABLE artifact ALTER COLUMN artifact_type SET NOT NULL;
|
||||
@ -127,10 +127,18 @@ func (a *abstractor) abstractManifestV2Metadata(artifact *artifact.Artifact, con
|
||||
}
|
||||
// use the "manifest.config.mediatype" as the media type of the artifact
|
||||
artifact.MediaType = manifest.Config.MediaType
|
||||
|
||||
if manifest.Annotations[wasm.AnnotationVariantKey] == wasm.AnnotationVariantValue || manifest.Annotations[wasm.AnnotationHandlerKey] == wasm.AnnotationHandlerValue {
|
||||
artifact.MediaType = wasm.MediaType
|
||||
}
|
||||
/*
|
||||
https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#listing-referrers
|
||||
For referrers list, if the artifactType is empty or missing in the image manifest, the value of artifactType MUST be set to the config descriptor mediaType value
|
||||
*/
|
||||
if manifest.ArtifactType != "" {
|
||||
artifact.ArtifactType = manifest.ArtifactType
|
||||
} else {
|
||||
artifact.ArtifactType = manifest.Config.MediaType
|
||||
}
|
||||
|
||||
// set size
|
||||
artifact.Size = int64(len(content)) + manifest.Config.Size
|
||||
@ -153,6 +161,16 @@ func (a *abstractor) abstractIndexMetadata(ctx context.Context, art *artifact.Ar
|
||||
return err
|
||||
}
|
||||
|
||||
/*
|
||||
https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#listing-referrers
|
||||
For referrers list, If the artifactType is empty or missing in an index, the artifactType MUST be omitted.
|
||||
*/
|
||||
if index.ArtifactType != "" {
|
||||
art.ArtifactType = index.ArtifactType
|
||||
} else {
|
||||
art.ArtifactType = ""
|
||||
}
|
||||
|
||||
// set annotations
|
||||
art.Annotations = index.Annotations
|
||||
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/docker/distribution"
|
||||
@ -175,7 +176,66 @@ var (
|
||||
"com.example.key1": "value1"
|
||||
}
|
||||
}`
|
||||
|
||||
OCIManifest = `{
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
||||
"config": {
|
||||
"mediaType": "application/vnd.example.config.v1+json",
|
||||
"digest": "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03",
|
||||
"size": 123
|
||||
},
|
||||
"layers": [
|
||||
{
|
||||
"mediaType": "application/vnd.example.data.v1.tar+gzip",
|
||||
"digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317",
|
||||
"size": 1234
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"com.example.key1": "value1"
|
||||
}
|
||||
}`
|
||||
OCIManifestWithArtifactType = `{
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
||||
"artifactType": "application/vnd.example+type",
|
||||
"config": {
|
||||
"mediaType": "application/vnd.example.config.v1+json",
|
||||
"digest": "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03",
|
||||
"size": 123
|
||||
},
|
||||
"layers": [
|
||||
{
|
||||
"mediaType": "application/vnd.example.data.v1.tar+gzip",
|
||||
"digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317",
|
||||
"size": 1234
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"com.example.key1": "value1"
|
||||
}
|
||||
}`
|
||||
OCIManifestWithEmptyConfig = `{
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
||||
"artifactType": "application/vnd.example+type",
|
||||
"config": {
|
||||
"mediaType": "application/vnd.oci.empty.v1+json",
|
||||
"digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
||||
"size": 2
|
||||
},
|
||||
"layers": [
|
||||
{
|
||||
"mediaType": "application/vnd.example+type",
|
||||
"digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317",
|
||||
"size": 1234
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"oci.opencontainers.image.created": "2023-01-02T03:04:05Z",
|
||||
"com.example.data": "payload"
|
||||
}
|
||||
}`
|
||||
index = `{
|
||||
"schemaVersion": 2,
|
||||
"manifests": [
|
||||
@ -202,6 +262,34 @@ var (
|
||||
"com.example.key1": "value1"
|
||||
}
|
||||
}`
|
||||
indexWithArtifactType = `{
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.oci.image.index.v1+json",
|
||||
"artifactType": "application/vnd.food.stand",
|
||||
"manifests": [
|
||||
{
|
||||
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
||||
"size": 7143,
|
||||
"digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
|
||||
"platform": {
|
||||
"architecture": "ppc64le",
|
||||
"os": "linux"
|
||||
}
|
||||
},
|
||||
{
|
||||
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
||||
"size": 7682,
|
||||
"digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
|
||||
"platform": {
|
||||
"architecture": "amd64",
|
||||
"os": "linux"
|
||||
}
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"com.example.key1": "value1"
|
||||
}
|
||||
}`
|
||||
)
|
||||
|
||||
type abstractorTestSuite struct {
|
||||
@ -267,6 +355,67 @@ func (a *abstractorTestSuite) TestAbstractMetadataOfV2Manifest() {
|
||||
a.Equal("value1", artifact.Annotations["com.example.key1"])
|
||||
}
|
||||
|
||||
// oci-spec v1
|
||||
func (a *abstractorTestSuite) TestAbstractMetadataOfOCIManifest() {
|
||||
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifest))
|
||||
a.Require().Nil(err)
|
||||
a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil)
|
||||
artifact := &artifact.Artifact{
|
||||
ID: 1,
|
||||
}
|
||||
a.processor.On("AbstractMetadata", mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||
err = a.abstractor.AbstractMetadata(context.TODO(), artifact)
|
||||
a.Require().Nil(err)
|
||||
a.Assert().Equal(int64(1), artifact.ID)
|
||||
a.Assert().Equal(v1.MediaTypeImageManifest, artifact.ManifestMediaType)
|
||||
a.Assert().Equal("application/vnd.example.config.v1+json", artifact.MediaType)
|
||||
a.Assert().Equal("application/vnd.example.config.v1+json", artifact.ArtifactType)
|
||||
a.Assert().Equal(int64(1916), artifact.Size)
|
||||
a.Require().Len(artifact.Annotations, 1)
|
||||
a.Equal("value1", artifact.Annotations["com.example.key1"])
|
||||
}
|
||||
|
||||
// oci-spec v1.1.0 with artifactType
|
||||
func (a *abstractorTestSuite) TestAbstractMetadataOfOCIManifestWithArtifactType() {
|
||||
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithArtifactType))
|
||||
a.Require().Nil(err)
|
||||
a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil)
|
||||
artifact := &artifact.Artifact{
|
||||
ID: 1,
|
||||
}
|
||||
a.processor.On("AbstractMetadata", mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||
err = a.abstractor.AbstractMetadata(context.TODO(), artifact)
|
||||
a.Require().Nil(err)
|
||||
a.Assert().Equal(int64(1), artifact.ID)
|
||||
a.Assert().Equal(v1.MediaTypeImageManifest, artifact.ManifestMediaType)
|
||||
a.Assert().Equal("application/vnd.example.config.v1+json", artifact.MediaType)
|
||||
a.Assert().Equal("application/vnd.example+type", artifact.ArtifactType)
|
||||
a.Assert().Equal(int64(1966), artifact.Size)
|
||||
a.Require().Len(artifact.Annotations, 1)
|
||||
a.Equal("value1", artifact.Annotations["com.example.key1"])
|
||||
}
|
||||
|
||||
// empty config with artifactType
|
||||
func (a *abstractorTestSuite) TestAbstractMetadataOfV2ManifestWithEmptyConfig() {
|
||||
// v1.MediaTypeImageManifest
|
||||
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithEmptyConfig))
|
||||
a.Require().Nil(err)
|
||||
a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil)
|
||||
artifact := &artifact.Artifact{
|
||||
ID: 1,
|
||||
}
|
||||
a.processor.On("AbstractMetadata", mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||
err = a.abstractor.AbstractMetadata(context.TODO(), artifact)
|
||||
a.Require().Nil(err)
|
||||
a.Assert().Equal(int64(1), artifact.ID)
|
||||
a.Assert().Equal(v1.MediaTypeImageManifest, artifact.ManifestMediaType)
|
||||
a.Assert().Equal(v1.MediaTypeEmptyJSON, artifact.MediaType)
|
||||
a.Assert().Equal("application/vnd.example+type", artifact.ArtifactType)
|
||||
a.Assert().Equal(int64(1880), artifact.Size)
|
||||
a.Require().Len(artifact.Annotations, 2)
|
||||
a.Equal("payload", artifact.Annotations["com.example.data"])
|
||||
}
|
||||
|
||||
// OCI index
|
||||
func (a *abstractorTestSuite) TestAbstractMetadataOfIndex() {
|
||||
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageIndex, []byte(index))
|
||||
@ -279,17 +428,41 @@ func (a *abstractorTestSuite) TestAbstractMetadataOfIndex() {
|
||||
artifact := &artifact.Artifact{
|
||||
ID: 1,
|
||||
}
|
||||
err = a.abstractor.AbstractMetadata(nil, artifact)
|
||||
err = a.abstractor.AbstractMetadata(context.TODO(), artifact)
|
||||
a.Require().Nil(err)
|
||||
a.Assert().Equal(int64(1), artifact.ID)
|
||||
a.Assert().Equal(v1.MediaTypeImageIndex, artifact.ManifestMediaType)
|
||||
a.Assert().Equal(v1.MediaTypeImageIndex, artifact.MediaType)
|
||||
a.Assert().Equal("", artifact.ArtifactType)
|
||||
a.Assert().Equal(int64(668), artifact.Size)
|
||||
a.Require().Len(artifact.Annotations, 1)
|
||||
a.Assert().Equal("value1", artifact.Annotations["com.example.key1"])
|
||||
a.Len(artifact.References, 2)
|
||||
}
|
||||
|
||||
func (a *abstractorTestSuite) TestAbstractMetadataOfIndexWithArtifactType() {
|
||||
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageIndex, []byte(indexWithArtifactType))
|
||||
a.Require().Nil(err)
|
||||
a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil)
|
||||
a.argMgr.On("GetByDigest", mock.Anything, mock.Anything, mock.Anything).Return(&artifact.Artifact{
|
||||
ID: 2,
|
||||
Size: 10,
|
||||
}, nil)
|
||||
artifact := &artifact.Artifact{
|
||||
ID: 1,
|
||||
}
|
||||
err = a.abstractor.AbstractMetadata(context.TODO(), artifact)
|
||||
a.Require().Nil(err)
|
||||
a.Assert().Equal(int64(1), artifact.ID)
|
||||
a.Assert().Equal(v1.MediaTypeImageIndex, artifact.ManifestMediaType)
|
||||
a.Assert().Equal(v1.MediaTypeImageIndex, artifact.MediaType)
|
||||
a.Assert().Equal("application/vnd.food.stand", artifact.ArtifactType)
|
||||
a.Assert().Equal(int64(801), artifact.Size)
|
||||
a.Require().Len(artifact.Annotations, 1)
|
||||
a.Assert().Equal("value1", artifact.Annotations["com.example.key1"])
|
||||
a.Len(artifact.References, 2)
|
||||
}
|
||||
|
||||
type unknownManifest struct{}
|
||||
|
||||
func (u *unknownManifest) References() []distribution.Descriptor {
|
||||
|
||||
@ -117,7 +117,31 @@ var (
|
||||
}
|
||||
]
|
||||
}`
|
||||
v2ManifestWithUnknownConfig = `{
|
||||
OCIManifestWithUnknownJsonConfig = `{
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
||||
"config": {
|
||||
"mediaType": "application/vnd.exmaple.config.v1+json",
|
||||
"digest": "sha256:48ef4a53c0770222d9752cd0588431dbda54667046208c79804e34c15c1579cd",
|
||||
"size": 129
|
||||
},
|
||||
"layers": [
|
||||
{
|
||||
"mediaType": "application/vnd.example.data.v1.tar+gzip",
|
||||
"digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317",
|
||||
"size": 1234
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"com.example.key1": "value1"
|
||||
}
|
||||
}`
|
||||
UnknownJsonConfig = `{
|
||||
"author": "yminer",
|
||||
"architecture": "amd64",
|
||||
"selfdefined": "true"
|
||||
}`
|
||||
OCIManifestWithUnknownConfig = `{
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
||||
"config": {
|
||||
@ -141,7 +165,30 @@ var (
|
||||
"newUnspecifiedField": null
|
||||
}
|
||||
}`
|
||||
unknownConfig = `{NHL Peanut Butter on my NHL bagel}`
|
||||
UnknownConfig = `{NHL Peanut Butter on my NHL bagel}`
|
||||
|
||||
OCIManifestWithEmptyConfig = `{
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
||||
"artifactType": "application/vnd.example+type",
|
||||
"config": {
|
||||
"mediaType": "application/vnd.oci.empty.v1+json",
|
||||
"digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
||||
"size": 2
|
||||
},
|
||||
"layers": [
|
||||
{
|
||||
"mediaType": "application/vnd.example+type",
|
||||
"digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317",
|
||||
"size": 1234
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"oci.opencontainers.image.created": "2023-01-02T03:04:05Z",
|
||||
"com.example.data": "payload"
|
||||
}
|
||||
}`
|
||||
emptyConfig = `{}`
|
||||
)
|
||||
|
||||
type defaultProcessorTestSuite struct {
|
||||
@ -190,6 +237,12 @@ func (d *defaultProcessorTestSuite) TestGetArtifactType() {
|
||||
typee = processor.GetArtifactType(nil, art)
|
||||
d.Equal("IMAGE", typee)
|
||||
|
||||
mediaType = "application/vnd.example.config.v1+json"
|
||||
art = &artifact.Artifact{MediaType: mediaType}
|
||||
processor = &defaultProcessor{}
|
||||
typee = processor.GetArtifactType(nil, art)
|
||||
d.Equal(ArtifactTypeUnknown, typee)
|
||||
|
||||
mediaType = "application/vnd.cncf.helm.chart.config.v1+json"
|
||||
art = &artifact.Artifact{MediaType: mediaType}
|
||||
processor = &defaultProcessor{}
|
||||
@ -229,19 +282,53 @@ func (d *defaultProcessorTestSuite) TestAbstractMetadata() {
|
||||
d.Len(art.ExtraAttrs, 12)
|
||||
}
|
||||
|
||||
func (d *defaultProcessorTestSuite) TestAbstractMetadataWithUnknownConfig() {
|
||||
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(v2ManifestWithUnknownConfig))
|
||||
func (d *defaultProcessorTestSuite) TestAbstractMetadataOfOCIManifesttWithUnknownJsonConfig() {
|
||||
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithUnknownJsonConfig))
|
||||
d.Require().Nil(err)
|
||||
manifestMediaType, content, err := manifest.Payload()
|
||||
d.Require().Nil(err)
|
||||
|
||||
configBlob := io.NopCloser(strings.NewReader(unknownConfig))
|
||||
d.regCli.On("PullBlob", mock.Anything, mock.Anything).Return(int64(0), configBlob, nil)
|
||||
art := &artifact.Artifact{ManifestMediaType: manifestMediaType}
|
||||
err = d.processor.AbstractMetadata(nil, art, content)
|
||||
configBlob := io.NopCloser(strings.NewReader(UnknownJsonConfig))
|
||||
metadata := map[string]interface{}{}
|
||||
err = json.NewDecoder(configBlob).Decode(&metadata)
|
||||
d.Require().Nil(err)
|
||||
|
||||
art := &artifact.Artifact{ManifestMediaType: manifestMediaType, MediaType: "application/vnd.example.config.v1+json"}
|
||||
|
||||
d.regCli.On("PullBlob", mock.Anything, mock.Anything).Return(int64(129), configBlob, nil)
|
||||
d.parser.On("Parse", context.TODO(), mock.AnythingOfType("*artifact.Artifact"), mock.AnythingOfType("[]byte")).Return(nil)
|
||||
err = d.processor.AbstractMetadata(context.TODO(), art, content)
|
||||
d.Require().Nil(err)
|
||||
d.Len(art.ExtraAttrs, 0)
|
||||
d.Len(unknownConfig, 35)
|
||||
d.NotEqual(art.ExtraAttrs, len(metadata))
|
||||
|
||||
}
|
||||
|
||||
func (d *defaultProcessorTestSuite) TestAbstractMetadataWithUnknownConfig() {
|
||||
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithUnknownConfig))
|
||||
d.Require().Nil(err)
|
||||
manifestMediaType, content, err := manifest.Payload()
|
||||
d.Require().Nil(err)
|
||||
|
||||
configBlob := io.NopCloser(strings.NewReader(UnknownConfig))
|
||||
d.regCli.On("PullBlob", mock.Anything, mock.Anything).Return(int64(0), configBlob, nil)
|
||||
art := &artifact.Artifact{ManifestMediaType: manifestMediaType, MediaType: "application/vnd.nhl.peanut.butter.bagel"}
|
||||
err = d.processor.AbstractMetadata(context.TODO(), art, content)
|
||||
d.Require().Nil(err)
|
||||
d.Len(art.ExtraAttrs, 0)
|
||||
}
|
||||
|
||||
func (d *defaultProcessorTestSuite) TestAbstractMetadataWithEmptyConfig() {
|
||||
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithEmptyConfig))
|
||||
d.Require().Nil(err)
|
||||
manifestMediaType, content, err := manifest.Payload()
|
||||
d.Require().Nil(err)
|
||||
|
||||
art := &artifact.Artifact{ManifestMediaType: manifestMediaType, MediaType: "application/vnd.oci.empty.v1+json"}
|
||||
err = d.processor.AbstractMetadata(context.TODO(), art, content)
|
||||
d.Assert().Equal(0, len(art.ExtraAttrs))
|
||||
d.Assert().Equal(2, len(emptyConfig))
|
||||
d.Require().Nil(err)
|
||||
}
|
||||
|
||||
func TestDefaultProcessorTestSuite(t *testing.T) {
|
||||
|
||||
@ -46,7 +46,7 @@ require (
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
|
||||
github.com/olekukonko/tablewriter v0.0.5
|
||||
github.com/opencontainers/go-digest v1.0.0
|
||||
github.com/opencontainers/image-spec v1.1.0-rc5
|
||||
github.com/opencontainers/image-spec v1.1.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/client_golang v1.17.0
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
|
||||
@ -514,8 +514,8 @@ github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
|
||||
github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI=
|
||||
github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
|
||||
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
|
||||
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
|
||||
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
|
||||
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
|
||||
@ -33,6 +33,7 @@ type Artifact struct {
|
||||
Type string `orm:"column(type)"` // image, chart or other OCI compatible
|
||||
MediaType string `orm:"column(media_type)"` // the media type of artifact
|
||||
ManifestMediaType string `orm:"column(manifest_media_type)"` // the media type of manifest/index
|
||||
ArtifactType string `orm:"colume(artifact_type)"` // the artifactType of manifest/index
|
||||
ProjectID int64 `orm:"column(project_id)"` // needed for quota
|
||||
RepositoryID int64 `orm:"column(repository_id)"`
|
||||
RepositoryName string `orm:"column(repository_name)"`
|
||||
|
||||
@ -34,6 +34,7 @@ type Artifact struct {
|
||||
Type string `json:"type"` // image, chart or other OCI compatible
|
||||
MediaType string `json:"media_type"` // the media type of artifact. Mostly, it's the value of `manifest.config.mediatype`
|
||||
ManifestMediaType string `json:"manifest_media_type"` // the media type of manifest/index
|
||||
ArtifactType string `json:"artifact_type"` // the artifactType of manifest/index
|
||||
ProjectID int64 `json:"project_id"`
|
||||
RepositoryID int64 `json:"repository_id"`
|
||||
RepositoryName string `json:"repository_name"`
|
||||
@ -63,6 +64,7 @@ func (a *Artifact) From(art *dao.Artifact) {
|
||||
a.Type = art.Type
|
||||
a.MediaType = art.MediaType
|
||||
a.ManifestMediaType = art.ManifestMediaType
|
||||
a.ArtifactType = art.ArtifactType
|
||||
a.ProjectID = art.ProjectID
|
||||
a.RepositoryID = art.RepositoryID
|
||||
a.RepositoryName = art.RepositoryName
|
||||
@ -92,6 +94,7 @@ func (a *Artifact) To() *dao.Artifact {
|
||||
Type: a.Type,
|
||||
MediaType: a.MediaType,
|
||||
ManifestMediaType: a.ManifestMediaType,
|
||||
ArtifactType: a.ArtifactType,
|
||||
ProjectID: a.ProjectID,
|
||||
RepositoryID: a.RepositoryID,
|
||||
RepositoryName: a.RepositoryName,
|
||||
|
||||
@ -37,6 +37,7 @@ func (m *modelTestSuite) TestArtifactFrom() {
|
||||
Type: "IMAGE",
|
||||
MediaType: "application/vnd.oci.image.config.v1+json",
|
||||
ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
|
||||
ArtifactType: "application/vnd.example+type",
|
||||
ProjectID: 1,
|
||||
RepositoryID: 1,
|
||||
Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
|
||||
@ -52,6 +53,7 @@ func (m *modelTestSuite) TestArtifactFrom() {
|
||||
assert.Equal(t, dbArt.Type, art.Type)
|
||||
assert.Equal(t, dbArt.MediaType, art.MediaType)
|
||||
assert.Equal(t, dbArt.ManifestMediaType, art.ManifestMediaType)
|
||||
assert.Equal(t, dbArt.ArtifactType, art.ArtifactType)
|
||||
assert.Equal(t, dbArt.ProjectID, art.ProjectID)
|
||||
assert.Equal(t, dbArt.RepositoryID, art.RepositoryID)
|
||||
assert.Equal(t, dbArt.Digest, art.Digest)
|
||||
@ -71,6 +73,7 @@ func (m *modelTestSuite) TestArtifactTo() {
|
||||
RepositoryID: 1,
|
||||
MediaType: "application/vnd.oci.image.config.v1+json",
|
||||
ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
|
||||
ArtifactType: "application/vnd.example+type",
|
||||
Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
|
||||
Size: 1024,
|
||||
PushTime: time.Now(),
|
||||
@ -87,6 +90,7 @@ func (m *modelTestSuite) TestArtifactTo() {
|
||||
assert.Equal(t, art.Type, dbArt.Type)
|
||||
assert.Equal(t, art.MediaType, dbArt.MediaType)
|
||||
assert.Equal(t, art.ManifestMediaType, dbArt.ManifestMediaType)
|
||||
assert.Equal(t, art.ArtifactType, dbArt.ArtifactType)
|
||||
assert.Equal(t, art.ProjectID, dbArt.ProjectID)
|
||||
assert.Equal(t, art.RepositoryID, dbArt.RepositoryID)
|
||||
assert.Equal(t, art.Digest, dbArt.Digest)
|
||||
|
||||
@ -269,7 +269,7 @@ func (suite *DaoTestSuite) TestFindBlobsShouldUnassociatedWithProject() {
|
||||
artifact1 := suite.DigestString()
|
||||
artifact2 := suite.DigestString()
|
||||
|
||||
sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world')`
|
||||
sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name, artifact_type) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world', 'artifact_type')`
|
||||
suite.ExecSQL(sql, artifact1, projectID, 10)
|
||||
suite.ExecSQL(sql, artifact2, projectID, 10)
|
||||
|
||||
|
||||
@ -130,7 +130,7 @@ func (suite *ManagerTestSuite) TestCleanupAssociationsForProject() {
|
||||
artifact1 := suite.DigestString()
|
||||
artifact2 := suite.DigestString()
|
||||
|
||||
sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world')`
|
||||
sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name, artifact_type) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world', 'artifact_type')`
|
||||
suite.ExecSQL(sql, artifact1, projectID, 10)
|
||||
suite.ExecSQL(sql, artifact2, projectID, 10)
|
||||
|
||||
@ -200,7 +200,7 @@ func (suite *ManagerTestSuite) TestFindBlobsShouldUnassociatedWithProject() {
|
||||
artifact1 := suite.DigestString()
|
||||
artifact2 := suite.DigestString()
|
||||
|
||||
sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world')`
|
||||
sql := `INSERT INTO artifact ("type", media_type, manifest_media_type, digest, project_id, repository_id, repository_name, artifact_type) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?, 'library/hello-world', 'artifact_type')`
|
||||
suite.ExecSQL(sql, artifact1, projectID, 11)
|
||||
suite.ExecSQL(sql, artifact2, projectID, 11)
|
||||
|
||||
|
||||
@ -49,12 +49,12 @@ func (suite *SecurityDaoTestSuite) SetupTest() {
|
||||
`delete from artifact_accessory`,
|
||||
`delete from artifact`,
|
||||
`insert into scan_report(uuid, digest, registration_uuid, mime_type, critical_cnt, high_cnt, medium_cnt, low_cnt, unknown_cnt, fixable_cnt) values('uuid', 'digest1001', 'ruuid', 'application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0', 50, 50, 50, 0, 0, 20)`,
|
||||
`insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon)
|
||||
values (1001, 1, 'library/hello-world', 'digest1001', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.docker.distribution.manifest.v2+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '');`,
|
||||
`insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon)
|
||||
values (1002, 1, 'library/hello-world', 'digest1002', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.oci.image.config.v1+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '');`,
|
||||
`insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon)
|
||||
values (1003, 1, 'library/hello-world', 'digest1003', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.oci.image.config.v1+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '');`,
|
||||
`insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon, artifact_type)
|
||||
values (1001, 1, 'library/hello-world', 'digest1001', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.docker.distribution.manifest.v2+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '', 'application/vnd.docker.container.image.v1+json');`,
|
||||
`insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon, artifact_type)
|
||||
values (1002, 1, 'library/hello-world', 'digest1002', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.oci.image.config.v1+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '', 'application/vnd.docker.container.image.v1+json');`,
|
||||
`insert into artifact (id, project_id, repository_name, digest, type, pull_time, push_time, repository_id, media_type, manifest_media_type, size, extra_attrs, annotations, icon, artifact_type)
|
||||
values (1003, 1, 'library/hello-world', 'digest1003', 'IMAGE', '2023-06-02 09:16:47.838778', '2023-06-02 01:45:55.050785', 1742, 'application/vnd.docker.container.image.v1+json', 'application/vnd.oci.image.config.v1+json', 4452, '{"architecture":"amd64","author":"","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"]},"created":"2023-05-04T17:37:03.872958712Z","os":"linux"}', null, '', 'application/vnd.docker.container.image.v1+json');`,
|
||||
`insert into tag (id, repository_id, artifact_id, name, push_time, pull_time) values (1001, 1742, 1001, 'latest', '2023-06-02 01:45:55.050785', '2023-06-02 09:16:47.838778')`,
|
||||
`INSERT INTO artifact_accessory (id, artifact_id, subject_artifact_id, type, size, digest, creation_time, subject_artifact_digest, subject_artifact_repo) VALUES (1001, 1002, 1, 'signature.cosign', 2109, 'sha256:08c64c0de2667abcf3974b4b75b82903f294680b81584318adc4826d0dcb7a9c', '2023-08-03 04:54:32.102928', 'sha256:a97a153152fcd6410bdf4fb64f5622ecf97a753f07dcc89dab14509d059736cf', 'library/nuxeo')`,
|
||||
`INSERT INTO artifact_reference (id, parent_id, child_id, child_digest, platform, urls, annotations) VALUES (1001, 1001, 1003, 'sha256:d2b2f2980e9ccc570e5726b56b54580f23a018b7b7314c9eaff7e5e479c78657', '{"architecture":"amd64","os":"linux"}', '', null)`,
|
||||
|
||||
@ -106,6 +106,15 @@ func Middleware() func(http.Handler) http.Handler {
|
||||
return err
|
||||
}
|
||||
|
||||
/*
|
||||
when an images is pushed, it could be
|
||||
1. single image (do nothing)
|
||||
2. an accesory image
|
||||
3. a subject image
|
||||
4. both as an accessory and a subject image
|
||||
and a subject image or accessory image could be pushed in either order
|
||||
*/
|
||||
|
||||
if mf.Subject != nil {
|
||||
subjectArt, err := artifact.Ctl.GetByReference(ctx, info.Repository, mf.Subject.Digest.String(), nil)
|
||||
if err != nil {
|
||||
@ -113,7 +122,7 @@ func Middleware() func(http.Handler) http.Handler {
|
||||
logger.Errorf("failed to get subject artifact: %s, error: %v", mf.Subject.Digest, err)
|
||||
return err
|
||||
}
|
||||
log.Debug("the subject of the signature doesn't exist.")
|
||||
log.Debug("the subject artifact doesn't exist.")
|
||||
}
|
||||
art, err := artifact.Ctl.GetByReference(ctx, info.Repository, info.Reference, nil)
|
||||
if err != nil {
|
||||
@ -128,7 +137,12 @@ func Middleware() func(http.Handler) http.Handler {
|
||||
Digest: art.Digest,
|
||||
}
|
||||
accData.Type = model.TypeSubject
|
||||
switch mf.Config.MediaType {
|
||||
// since oci-spec 1.1, image type may from artifactType if presents, otherwise would be Config.MediaType
|
||||
fromType := mf.Config.MediaType
|
||||
if mf.ArtifactType != "" {
|
||||
fromType = mf.ArtifactType
|
||||
}
|
||||
switch fromType {
|
||||
case ocispec.MediaTypeImageConfig, schema2.MediaTypeImageConfig:
|
||||
if isNydusImage(mf) {
|
||||
accData.Type = model.TypeNydusAccelerator
|
||||
@ -152,18 +166,18 @@ func Middleware() func(http.Handler) http.Handler {
|
||||
// when subject artifact is pushed after accessory artifact, current subject artifact do not exist.
|
||||
// so we use reference manifest subject digest instead of subjectArt.Digest
|
||||
w.Header().Set("OCI-Subject", mf.Subject.Digest.String())
|
||||
} else {
|
||||
}
|
||||
|
||||
// check if images is a Subject artifact
|
||||
digest := digest.FromBytes(body)
|
||||
accs, err := accessory.Mgr.List(ctx, q.New(q.KeyWords{"SubjectArtifactDigest": digest, "SubjectArtifactRepo": info.Repository}))
|
||||
if err != nil {
|
||||
logger.Errorf("failed to list accessory artifact: %s, error: %v", digest, err)
|
||||
return err
|
||||
}
|
||||
if len(accs) > 0 {
|
||||
// In certain cases, the OCI client may push the subject artifact and accessory in either order.
|
||||
// Therefore, it is necessary to handle situations where the client pushes the accessory ahead of the subject artifact.
|
||||
digest := digest.FromBytes(body)
|
||||
accs, err := accessory.Mgr.List(ctx, q.New(q.KeyWords{"SubjectArtifactDigest": digest, "SubjectArtifactRepo": info.Repository}))
|
||||
if err != nil {
|
||||
logger.Errorf("failed to list accessory artifact: %s, error: %v", digest, err)
|
||||
return err
|
||||
}
|
||||
if len(accs) <= 0 {
|
||||
return nil
|
||||
}
|
||||
art, err := artifact.Ctl.GetByReference(ctx, info.Repository, digest.String(), nil)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to list artifact: %s, error: %v", digest, err)
|
||||
|
||||
@ -89,11 +89,11 @@ func (r *referrersHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
Size: accArt.Size,
|
||||
Digest: digest.Digest(accArt.Digest),
|
||||
Annotations: accArt.Annotations,
|
||||
ArtifactType: accArt.MediaType,
|
||||
ArtifactType: accArt.ArtifactType,
|
||||
}
|
||||
// filter by the artifactType since the artifactType is actually the config media type of the artifact.
|
||||
// filter use accArt.ArtifactType as artifactType
|
||||
if at != "" {
|
||||
if accArt.MediaType == at {
|
||||
if accArt.ArtifactType == at {
|
||||
mfs = append(mfs, mf)
|
||||
}
|
||||
} else {
|
||||
|
||||
@ -37,7 +37,8 @@ func TestReferrersHandlerOK(t *testing.T) {
|
||||
Return(&artifact.Artifact{
|
||||
Digest: digestVal,
|
||||
ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
|
||||
MediaType: "application/vnd.example.main",
|
||||
MediaType: "application/vnd.example.sbom",
|
||||
ArtifactType: "application/vnd.example+type",
|
||||
Size: 1000,
|
||||
Annotations: map[string]string{
|
||||
"name": "test-image",
|
||||
@ -72,8 +73,8 @@ func TestReferrersHandlerOK(t *testing.T) {
|
||||
}
|
||||
index := &ocispec.Index{}
|
||||
json.Unmarshal([]byte(rec.Body.String()), index)
|
||||
if index.Manifests[0].ArtifactType != "application/vnd.example.main" {
|
||||
t.Errorf("Expected response body %s, but got %s", "application/vnd.example.main", rec.Body.String())
|
||||
if index.Manifests[0].ArtifactType != "application/vnd.example+type" {
|
||||
t.Errorf("Expected response body %s, but got %s", "application/vnd.example+type", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user