mirror of
https://github.com/goharbor/harbor.git
synced 2024-09-06 02:58:04 +02:00
Merge pull request #12525 from hyy0322/enhance-default-processor
Feat: Enhance default processor to handle user-defined artifact like Machine Learning artifacts
This commit is contained in:
commit
ed80baedc2
@ -113,5 +113,6 @@ END $$;
|
|||||||
ALTER TABLE schedule DROP COLUMN IF EXISTS job_id;
|
ALTER TABLE schedule DROP COLUMN IF EXISTS job_id;
|
||||||
ALTER TABLE schedule DROP COLUMN IF EXISTS status;
|
ALTER TABLE schedule DROP COLUMN IF EXISTS status;
|
||||||
|
|
||||||
/*replication quay.io update vendor type*/
|
|
||||||
UPDATE registry SET type = 'quay' WHERE type = 'quay-io';
|
UPDATE registry SET type = 'quay' WHERE type = 'quay-io';
|
||||||
|
|
||||||
|
ALTER TABLE artifact ADD COLUMN icon varchar(255);
|
||||||
|
@ -62,17 +62,17 @@ func (a *abstractor) AbstractMetadata(ctx context.Context, artifact *artifact.Ar
|
|||||||
case "", "application/json", schema1.MediaTypeSignedManifest:
|
case "", "application/json", schema1.MediaTypeSignedManifest:
|
||||||
a.abstractManifestV1Metadata(artifact)
|
a.abstractManifestV1Metadata(artifact)
|
||||||
case v1.MediaTypeImageManifest, schema2.MediaTypeManifest:
|
case v1.MediaTypeImageManifest, schema2.MediaTypeManifest:
|
||||||
if err = a.abstractManifestV2Metadata(content, artifact); err != nil {
|
if err = a.abstractManifestV2Metadata(artifact, content); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
case v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList:
|
case v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList:
|
||||||
if err = a.abstractIndexMetadata(ctx, content, artifact); err != nil {
|
if err = a.abstractIndexMetadata(ctx, artifact, content); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("unsupported manifest media type: %s", artifact.ManifestMediaType)
|
return fmt.Errorf("unsupported manifest media type: %s", artifact.ManifestMediaType)
|
||||||
}
|
}
|
||||||
return processor.Get(artifact.MediaType).AbstractMetadata(ctx, content, artifact)
|
return processor.Get(artifact.MediaType).AbstractMetadata(ctx, artifact, content)
|
||||||
}
|
}
|
||||||
|
|
||||||
// the artifact is enveloped by docker manifest v1
|
// the artifact is enveloped by docker manifest v1
|
||||||
@ -86,7 +86,7 @@ func (a *abstractor) abstractManifestV1Metadata(artifact *artifact.Artifact) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// the artifact is enveloped by OCI manifest or docker manifest v2
|
// the artifact is enveloped by OCI manifest or docker manifest v2
|
||||||
func (a *abstractor) abstractManifestV2Metadata(content []byte, artifact *artifact.Artifact) error {
|
func (a *abstractor) abstractManifestV2Metadata(artifact *artifact.Artifact, content []byte) error {
|
||||||
manifest := &v1.Manifest{}
|
manifest := &v1.Manifest{}
|
||||||
if err := json.Unmarshal(content, manifest); err != nil {
|
if err := json.Unmarshal(content, manifest); err != nil {
|
||||||
return err
|
return err
|
||||||
@ -104,7 +104,7 @@ func (a *abstractor) abstractManifestV2Metadata(content []byte, artifact *artifa
|
|||||||
}
|
}
|
||||||
|
|
||||||
// the artifact is enveloped by OCI index or docker manifest list
|
// the artifact is enveloped by OCI index or docker manifest list
|
||||||
func (a *abstractor) abstractIndexMetadata(ctx context.Context, content []byte, art *artifact.Artifact) error {
|
func (a *abstractor) abstractIndexMetadata(ctx context.Context, art *artifact.Artifact, content []byte) error {
|
||||||
// the identity of index is still in progress, we use the manifest mediaType
|
// the identity of index is still in progress, we use the manifest mediaType
|
||||||
// as the media type of artifact
|
// as the media type of artifact
|
||||||
art.MediaType = art.ManifestMediaType
|
art.MediaType = art.ManifestMediaType
|
||||||
|
@ -15,16 +15,20 @@
|
|||||||
package artifact
|
package artifact
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/controller/artifact/processor"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/artifact"
|
||||||
|
"github.com/goharbor/harbor/src/testing/mock"
|
||||||
|
tart "github.com/goharbor/harbor/src/testing/pkg/artifact"
|
||||||
|
tpro "github.com/goharbor/harbor/src/testing/pkg/processor"
|
||||||
|
"github.com/goharbor/harbor/src/testing/pkg/registry"
|
||||||
|
|
||||||
"github.com/docker/distribution"
|
"github.com/docker/distribution"
|
||||||
"github.com/docker/distribution/manifest/schema1"
|
"github.com/docker/distribution/manifest/schema1"
|
||||||
"github.com/docker/distribution/manifest/schema2"
|
"github.com/docker/distribution/manifest/schema2"
|
||||||
"github.com/goharbor/harbor/src/controller/artifact/processor"
|
|
||||||
"github.com/goharbor/harbor/src/pkg/artifact"
|
|
||||||
tart "github.com/goharbor/harbor/src/testing/pkg/artifact"
|
|
||||||
"github.com/goharbor/harbor/src/testing/pkg/registry"
|
|
||||||
v1 "github.com/opencontainers/image-spec/specs-go/v1"
|
v1 "github.com/opencontainers/image-spec/specs-go/v1"
|
||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
"testing"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -203,6 +207,7 @@ type abstractorTestSuite struct {
|
|||||||
argMgr *tart.FakeManager
|
argMgr *tart.FakeManager
|
||||||
regCli *registry.FakeClient
|
regCli *registry.FakeClient
|
||||||
abstractor *abstractor
|
abstractor *abstractor
|
||||||
|
processor *tpro.Processor
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *abstractorTestSuite) SetupTest() {
|
func (a *abstractorTestSuite) SetupTest() {
|
||||||
@ -212,8 +217,10 @@ func (a *abstractorTestSuite) SetupTest() {
|
|||||||
artMgr: a.argMgr,
|
artMgr: a.argMgr,
|
||||||
regCli: a.regCli,
|
regCli: a.regCli,
|
||||||
}
|
}
|
||||||
|
a.processor = &tpro.Processor{}
|
||||||
// clear all registered processors
|
// clear all registered processors
|
||||||
processor.Registry = map[string]processor.Processor{}
|
processor.Registry = map[string]processor.Processor{}
|
||||||
|
processor.Registry[schema2.MediaTypeImageConfig] = a.processor
|
||||||
}
|
}
|
||||||
|
|
||||||
// docker manifest v1
|
// docker manifest v1
|
||||||
@ -240,6 +247,7 @@ func (a *abstractorTestSuite) TestAbstractMetadataOfV2Manifest() {
|
|||||||
artifact := &artifact.Artifact{
|
artifact := &artifact.Artifact{
|
||||||
ID: 1,
|
ID: 1,
|
||||||
}
|
}
|
||||||
|
a.processor.On("AbstractMetadata", mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||||
err = a.abstractor.AbstractMetadata(nil, artifact)
|
err = a.abstractor.AbstractMetadata(nil, artifact)
|
||||||
a.Require().Nil(err)
|
a.Require().Nil(err)
|
||||||
a.Assert().Equal(int64(1), artifact.ID)
|
a.Assert().Equal(int64(1), artifact.ID)
|
||||||
|
77
src/controller/artifact/annotation/parser.go
Normal file
77
src/controller/artifact/annotation/parser.go
Normal file
@ -0,0 +1,77 @@
|
|||||||
|
package annotation
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/lib/log"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/artifact"
|
||||||
|
reg "github.com/goharbor/harbor/src/pkg/registry"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// GIF is icon content type image/gif
|
||||||
|
GIF = "image/gif"
|
||||||
|
// PNG is icon content type image/png
|
||||||
|
PNG = "image/png"
|
||||||
|
// JPEG is icon content type image/jpeg
|
||||||
|
JPEG = "image/jpeg"
|
||||||
|
|
||||||
|
// AnnotationPrefix is the prefix of annotation
|
||||||
|
AnnotationPrefix = "io.goharbor.artifact"
|
||||||
|
|
||||||
|
// SkipList is the key word of skip-list annotation
|
||||||
|
SkipList = "skip-list"
|
||||||
|
// Icon is the key word of icon annotation
|
||||||
|
Icon = "icon"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// registry for registered annotation parsers
|
||||||
|
registry = map[string]Parser{}
|
||||||
|
|
||||||
|
// sortedAnnotationVersionList define the order of AnnotationParser from low to high version.
|
||||||
|
// Low version annotation parser will parser annotation first.
|
||||||
|
sortedAnnotationVersionList = make([]string, 0)
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
v1alpha1Parser := &v1alpha1Parser{
|
||||||
|
regCli: reg.Cli,
|
||||||
|
}
|
||||||
|
RegisterAnnotationParser(v1alpha1Parser, V1alpha1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewParser creates a new annotation parser
|
||||||
|
func NewParser() Parser {
|
||||||
|
return &parser{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parser parses annotations in artifact manifest
|
||||||
|
type Parser interface {
|
||||||
|
// Parse parses annotations in artifact manifest, abstracts data from artifact config layer into the artifact model
|
||||||
|
Parse(ctx context.Context, artifact *artifact.Artifact, manifest []byte) (err error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type parser struct{}
|
||||||
|
|
||||||
|
func (p *parser) Parse(ctx context.Context, artifact *artifact.Artifact, manifest []byte) (err error) {
|
||||||
|
for _, annotationVersion := range sortedAnnotationVersionList {
|
||||||
|
err = GetAnnotationParser(annotationVersion).Parse(ctx, artifact, manifest)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RegisterAnnotationParser register annotation parser
|
||||||
|
func RegisterAnnotationParser(parser Parser, version string) {
|
||||||
|
registry[version] = parser
|
||||||
|
sortedAnnotationVersionList = append(sortedAnnotationVersionList, version)
|
||||||
|
log.Infof("the annotation parser to parser artifact annotation version %s registered", version)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAnnotationParser register annotation parser
|
||||||
|
func GetAnnotationParser(version string) Parser {
|
||||||
|
return registry[version]
|
||||||
|
}
|
41
src/controller/artifact/annotation/parser_test.go
Normal file
41
src/controller/artifact/annotation/parser_test.go
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
package annotation
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
fp "github.com/goharbor/harbor/src/testing/pkg/parser"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
)
|
||||||
|
|
||||||
|
type parserTestSuite struct {
|
||||||
|
suite.Suite
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *parserTestSuite) SetupTest() {
|
||||||
|
registry = map[string]Parser{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *parserTestSuite) TestRegisterAnnotationParser() {
|
||||||
|
// success
|
||||||
|
version := "v1alpha1"
|
||||||
|
parser := &fp.Parser{}
|
||||||
|
RegisterAnnotationParser(parser, version)
|
||||||
|
p.Equal(map[string]Parser{version: parser}, registry)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *parserTestSuite) TestGetAnnotationParser() {
|
||||||
|
// register the parser
|
||||||
|
version := "v1alpha1"
|
||||||
|
RegisterAnnotationParser(&fp.Parser{}, "v1alpha1")
|
||||||
|
|
||||||
|
// get the parser
|
||||||
|
parser := GetAnnotationParser(version)
|
||||||
|
p.Require().NotNil(parser)
|
||||||
|
_, ok := parser.(*fp.Parser)
|
||||||
|
p.True(ok)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessorTestSuite(t *testing.T) {
|
||||||
|
suite.Run(t, &parserTestSuite{})
|
||||||
|
}
|
99
src/controller/artifact/annotation/v1alpha1.go
Normal file
99
src/controller/artifact/annotation/v1alpha1.go
Normal file
@ -0,0 +1,99 @@
|
|||||||
|
package annotation
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/lib/errors"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/artifact"
|
||||||
|
reg "github.com/goharbor/harbor/src/pkg/registry"
|
||||||
|
|
||||||
|
"github.com/docker/distribution/manifest/schema2"
|
||||||
|
v1 "github.com/opencontainers/image-spec/specs-go/v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// V1alpha1 is the version of annotation parser
|
||||||
|
V1alpha1 = "v1alpha1"
|
||||||
|
)
|
||||||
|
|
||||||
|
type v1alpha1Parser struct {
|
||||||
|
regCli reg.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *v1alpha1Parser) Parse(ctx context.Context, artifact *artifact.Artifact, manifest []byte) error {
|
||||||
|
if artifact.ManifestMediaType != v1.MediaTypeImageManifest && artifact.ManifestMediaType != schema2.MediaTypeManifest {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// get manifest
|
||||||
|
mani := &v1.Manifest{}
|
||||||
|
if err := json.Unmarshal(manifest, mani); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// parse skip-list annotation io.goharor.artifact.v1alpha1.skip-list
|
||||||
|
parseV1alpha1SkipList(artifact, mani)
|
||||||
|
|
||||||
|
// parse icon annotation io.goharbor.artifact.v1alpha1.icon
|
||||||
|
err := parseV1alpha1Icon(artifact, mani, p.regCli)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseV1alpha1SkipList(artifact *artifact.Artifact, manifest *v1.Manifest) {
|
||||||
|
metadata := artifact.ExtraAttrs
|
||||||
|
skipListAnnotationKey := fmt.Sprintf("%s.%s.%s", AnnotationPrefix, V1alpha1, SkipList)
|
||||||
|
skipList, ok := manifest.Config.Annotations[skipListAnnotationKey]
|
||||||
|
if ok {
|
||||||
|
skipKeyList := strings.Split(skipList, ",")
|
||||||
|
for _, skipKey := range skipKeyList {
|
||||||
|
delete(metadata, skipKey)
|
||||||
|
}
|
||||||
|
artifact.ExtraAttrs = metadata
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseV1alpha1Icon(artifact *artifact.Artifact, manifest *v1.Manifest, reg reg.Client) error {
|
||||||
|
iconAnnotationKey := fmt.Sprintf("%s.%s.%s", AnnotationPrefix, V1alpha1, Icon)
|
||||||
|
var iconDigest string
|
||||||
|
for _, layer := range manifest.Layers {
|
||||||
|
_, ok := layer.Annotations[iconAnnotationKey]
|
||||||
|
if ok {
|
||||||
|
iconDigest = layer.Digest.String()
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if iconDigest == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// pull icon layer
|
||||||
|
_, icon, err := reg.PullBlob(artifact.RepositoryName, iconDigest)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// check the size of the size <= 1MB
|
||||||
|
data, err := ioutil.ReadAll(io.LimitReader(icon, 1<<20))
|
||||||
|
if err != nil {
|
||||||
|
if err == io.EOF {
|
||||||
|
return errors.New(nil).WithCode(errors.BadRequestCode).WithMessage("the maximum size of the icon is 1MB")
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// check the content type
|
||||||
|
contentType := http.DetectContentType(data)
|
||||||
|
switch contentType {
|
||||||
|
case GIF, PNG, JPEG:
|
||||||
|
default:
|
||||||
|
return errors.New(nil).WithCode(errors.BadRequestCode).WithMessage("unsupported content type: %s", contentType)
|
||||||
|
}
|
||||||
|
artifact.Icon = iconDigest
|
||||||
|
return nil
|
||||||
|
}
|
255
src/controller/artifact/annotation/v1alpha1_test.go
Normal file
255
src/controller/artifact/annotation/v1alpha1_test.go
Normal file
File diff suppressed because one or more lines are too long
@ -185,7 +185,7 @@ func (c *controller) ensureArtifact(ctx context.Context, repository, digest stri
|
|||||||
}
|
}
|
||||||
|
|
||||||
// populate the artifact type
|
// populate the artifact type
|
||||||
artifact.Type = processor.Get(artifact.MediaType).GetArtifactType()
|
artifact.Type = processor.Get(artifact.MediaType).GetArtifactType(ctx, artifact)
|
||||||
|
|
||||||
// create it
|
// create it
|
||||||
// use orm.WithTransaction here to avoid the issue:
|
// use orm.WithTransaction here to avoid the issue:
|
||||||
@ -600,7 +600,7 @@ func (c *controller) populateLabels(ctx context.Context, art *Artifact) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *controller) populateAdditionLinks(ctx context.Context, artifact *Artifact) {
|
func (c *controller) populateAdditionLinks(ctx context.Context, artifact *Artifact) {
|
||||||
types := processor.Get(artifact.MediaType).ListAdditionTypes()
|
types := processor.Get(artifact.MediaType).ListAdditionTypes(ctx, &artifact.Artifact)
|
||||||
if len(types) > 0 {
|
if len(types) > 0 {
|
||||||
version := lib.GetAPIVersion(ctx)
|
version := lib.GetAPIVersion(ctx)
|
||||||
for _, t := range types {
|
for _, t := range types {
|
||||||
|
@ -36,7 +36,7 @@ type IndexProcessor struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// AbstractMetadata abstracts metadata of artifact
|
// AbstractMetadata abstracts metadata of artifact
|
||||||
func (m *IndexProcessor) AbstractMetadata(ctx context.Context, content []byte, artifact *artifact.Artifact) error {
|
func (m *IndexProcessor) AbstractMetadata(ctx context.Context, artifact *artifact.Artifact, content []byte) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -47,11 +47,11 @@ func (m *IndexProcessor) AbstractAddition(ctx context.Context, artifact *artifac
|
|||||||
}
|
}
|
||||||
|
|
||||||
// GetArtifactType returns the artifact type
|
// GetArtifactType returns the artifact type
|
||||||
func (m *IndexProcessor) GetArtifactType() string {
|
func (m *IndexProcessor) GetArtifactType(ctx context.Context, artifact *artifact.Artifact) string {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListAdditionTypes returns the supported addition types
|
// ListAdditionTypes returns the supported addition types
|
||||||
func (m *IndexProcessor) ListAdditionTypes() []string {
|
func (m *IndexProcessor) ListAdditionTypes(ctx context.Context, artifact *artifact.Artifact) []string {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -40,7 +40,7 @@ type ManifestProcessor struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// AbstractMetadata abstracts metadata of artifact
|
// AbstractMetadata abstracts metadata of artifact
|
||||||
func (m *ManifestProcessor) AbstractMetadata(ctx context.Context, content []byte, artifact *artifact.Artifact) error {
|
func (m *ManifestProcessor) AbstractMetadata(ctx context.Context, artifact *artifact.Artifact, content []byte) error {
|
||||||
// get manifest
|
// get manifest
|
||||||
manifest := &v1.Manifest{}
|
manifest := &v1.Manifest{}
|
||||||
if err := json.Unmarshal(content, manifest); err != nil {
|
if err := json.Unmarshal(content, manifest); err != nil {
|
||||||
@ -79,11 +79,11 @@ func (m *ManifestProcessor) AbstractAddition(ctx context.Context, artifact *arti
|
|||||||
}
|
}
|
||||||
|
|
||||||
// GetArtifactType returns the artifact type
|
// GetArtifactType returns the artifact type
|
||||||
func (m *ManifestProcessor) GetArtifactType() string {
|
func (m *ManifestProcessor) GetArtifactType(ctx context.Context, artifact *artifact.Artifact) string {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListAdditionTypes returns the supported addition types
|
// ListAdditionTypes returns the supported addition types
|
||||||
func (m *ManifestProcessor) ListAdditionTypes() []string {
|
func (m *ManifestProcessor) ListAdditionTypes(ctx context.Context, artifact *artifact.Artifact) []string {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -138,7 +138,7 @@ func (m *manifestTestSuite) TestAbstractMetadata() {
|
|||||||
art := &artifact.Artifact{}
|
art := &artifact.Artifact{}
|
||||||
|
|
||||||
m.regCli.On("PullBlob").Return(0, ioutil.NopCloser(strings.NewReader(config)), nil)
|
m.regCli.On("PullBlob").Return(0, ioutil.NopCloser(strings.NewReader(config)), nil)
|
||||||
m.processor.AbstractMetadata(nil, []byte(manifest), art)
|
m.processor.AbstractMetadata(nil, art, []byte(manifest))
|
||||||
m.Len(art.ExtraAttrs, 9)
|
m.Len(art.ExtraAttrs, 9)
|
||||||
|
|
||||||
// reset the mock
|
// reset the mock
|
||||||
@ -148,7 +148,7 @@ func (m *manifestTestSuite) TestAbstractMetadata() {
|
|||||||
m.processor.properties = []string{"os"}
|
m.processor.properties = []string{"os"}
|
||||||
art = &artifact.Artifact{}
|
art = &artifact.Artifact{}
|
||||||
m.regCli.On("PullBlob").Return(0, ioutil.NopCloser(strings.NewReader(config)), nil)
|
m.regCli.On("PullBlob").Return(0, ioutil.NopCloser(strings.NewReader(config)), nil)
|
||||||
m.processor.AbstractMetadata(nil, []byte(manifest), art)
|
m.processor.AbstractMetadata(nil, art, []byte(manifest))
|
||||||
m.Require().Len(art.ExtraAttrs, 1)
|
m.Require().Len(art.ExtraAttrs, 1)
|
||||||
m.Equal("linux", art.ExtraAttrs["os"])
|
m.Equal("linux", art.ExtraAttrs["os"])
|
||||||
}
|
}
|
||||||
|
@ -121,10 +121,10 @@ func (p *processor) AbstractAddition(ctx context.Context, artifact *artifact.Art
|
|||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *processor) GetArtifactType() string {
|
func (p *processor) GetArtifactType(ctx context.Context, artifact *artifact.Artifact) string {
|
||||||
return ArtifactTypeChart
|
return ArtifactTypeChart
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *processor) ListAdditionTypes() []string {
|
func (p *processor) ListAdditionTypes(ctx context.Context, artifact *artifact.Artifact) []string {
|
||||||
return []string{AdditionTypeValues, AdditionTypeReadme, AdditionTypeDependencies}
|
return []string{AdditionTypeValues, AdditionTypeReadme, AdditionTypeDependencies}
|
||||||
}
|
}
|
||||||
|
@ -128,11 +128,11 @@ func (p *processorTestSuite) TestAbstractAddition() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (p *processorTestSuite) TestGetArtifactType() {
|
func (p *processorTestSuite) TestGetArtifactType() {
|
||||||
p.Assert().Equal(ArtifactTypeChart, p.processor.GetArtifactType())
|
p.Assert().Equal(ArtifactTypeChart, p.processor.GetArtifactType(nil, nil))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *processorTestSuite) TestListAdditionTypes() {
|
func (p *processorTestSuite) TestListAdditionTypes() {
|
||||||
additions := p.processor.ListAdditionTypes()
|
additions := p.processor.ListAdditionTypes(nil, nil)
|
||||||
p.EqualValues([]string{AdditionTypeValues, AdditionTypeReadme, AdditionTypeDependencies}, additions)
|
p.EqualValues([]string{AdditionTypeValues, AdditionTypeReadme, AdditionTypeDependencies}, additions)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -45,7 +45,7 @@ type processor struct {
|
|||||||
manifestProcessor *base.ManifestProcessor
|
manifestProcessor *base.ManifestProcessor
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *processor) AbstractMetadata(ctx context.Context, manifest []byte, art *artifact.Artifact) error {
|
func (p *processor) AbstractMetadata(ctx context.Context, art *artifact.Artifact, manifest []byte) error {
|
||||||
cfgManiDgt := ""
|
cfgManiDgt := ""
|
||||||
// try to get the digest of the manifest that the config layer is referenced by
|
// try to get the digest of the manifest that the config layer is referenced by
|
||||||
for _, reference := range art.References {
|
for _, reference := range art.References {
|
||||||
@ -69,9 +69,9 @@ func (p *processor) AbstractMetadata(ctx context.Context, manifest []byte, art *
|
|||||||
}
|
}
|
||||||
|
|
||||||
// abstract the metadata from config layer
|
// abstract the metadata from config layer
|
||||||
return p.manifestProcessor.AbstractMetadata(ctx, payload, art)
|
return p.manifestProcessor.AbstractMetadata(ctx, art, payload)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *processor) GetArtifactType() string {
|
func (p *processor) GetArtifactType(ctx context.Context, artifact *artifact.Artifact) string {
|
||||||
return ArtifactTypeCNAB
|
return ArtifactTypeCNAB
|
||||||
}
|
}
|
||||||
|
@ -93,7 +93,7 @@ func (p *processorTestSuite) TestAbstractMetadata() {
|
|||||||
p.Require().Nil(err)
|
p.Require().Nil(err)
|
||||||
p.regCli.On("PullManifest").Return(mani, "", nil)
|
p.regCli.On("PullManifest").Return(mani, "", nil)
|
||||||
p.regCli.On("PullBlob").Return(0, ioutil.NopCloser(strings.NewReader(config)), nil)
|
p.regCli.On("PullBlob").Return(0, ioutil.NopCloser(strings.NewReader(config)), nil)
|
||||||
err = p.processor.AbstractMetadata(nil, nil, art)
|
err = p.processor.AbstractMetadata(nil, art, nil)
|
||||||
p.Require().Nil(err)
|
p.Require().Nil(err)
|
||||||
p.Len(art.ExtraAttrs, 7)
|
p.Len(art.ExtraAttrs, 7)
|
||||||
p.Equal("0.1.1", art.ExtraAttrs["version"].(string))
|
p.Equal("0.1.1", art.ExtraAttrs["version"].(string))
|
||||||
@ -101,7 +101,7 @@ func (p *processorTestSuite) TestAbstractMetadata() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (p *processorTestSuite) TestGetArtifactType() {
|
func (p *processorTestSuite) TestGetArtifactType() {
|
||||||
p.Assert().Equal(ArtifactTypeCNAB, p.processor.GetArtifactType())
|
p.Assert().Equal(ArtifactTypeCNAB, p.processor.GetArtifactType(nil, nil))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProcessorTestSuite(t *testing.T) {
|
func TestProcessorTestSuite(t *testing.T) {
|
||||||
|
@ -16,43 +16,124 @@ package processor
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"github.com/goharbor/harbor/src/lib/errors"
|
"encoding/json"
|
||||||
"github.com/goharbor/harbor/src/pkg/artifact"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
// annotation parsers will be registered
|
||||||
|
"github.com/goharbor/harbor/src/controller/artifact/annotation"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/lib/errors"
|
||||||
|
"github.com/goharbor/harbor/src/lib/log"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/artifact"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/registry"
|
||||||
|
|
||||||
|
"github.com/docker/distribution/manifest/schema2"
|
||||||
|
v1 "github.com/opencontainers/image-spec/specs-go/v1"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ArtifactTypeUnknown defines the type for the unknown artifacts
|
const (
|
||||||
const ArtifactTypeUnknown = "UNKNOWN"
|
// ArtifactTypeUnknown defines the type for the unknown artifacts
|
||||||
|
ArtifactTypeUnknown = "UNKNOWN"
|
||||||
|
// DefaultIconDigest defines default icon layer digest
|
||||||
|
DefaultIconDigest = "sha256:da834479c923584f4cbcdecc0dac61f32bef1d51e8aae598cf16bd154efab49f"
|
||||||
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
// DefaultProcessor is to process artifact which has no specific processor
|
||||||
|
DefaultProcessor = &defaultProcessor{regCli: registry.Cli}
|
||||||
|
|
||||||
artifactTypeRegExp = regexp.MustCompile(`^application/vnd\.[^.]*\.(.*)\.config\.[^.]*\+json$`)
|
artifactTypeRegExp = regexp.MustCompile(`^application/vnd\.[^.]*\.(.*)\.config\.[^.]*\+json$`)
|
||||||
)
|
)
|
||||||
|
|
||||||
// the default processor to process artifact
|
// the default processor to process artifact
|
||||||
// currently, it only tries to parse the artifact type from media type
|
|
||||||
type defaultProcessor struct {
|
type defaultProcessor struct {
|
||||||
mediaType string
|
regCli registry.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *defaultProcessor) GetArtifactType() string {
|
func (d *defaultProcessor) GetArtifactType(ctx context.Context, artifact *artifact.Artifact) string {
|
||||||
// try to parse the type from the media type
|
// try to parse the type from the media type
|
||||||
strs := artifactTypeRegExp.FindStringSubmatch(d.mediaType)
|
strs := artifactTypeRegExp.FindStringSubmatch(artifact.MediaType)
|
||||||
if len(strs) == 2 {
|
if len(strs) == 2 {
|
||||||
return strings.ToUpper(strs[1])
|
return strings.ToUpper(strs[1])
|
||||||
}
|
}
|
||||||
// can not get the artifact type from the media type, return unknown
|
// can not get the artifact type from the media type, return unknown
|
||||||
return ArtifactTypeUnknown
|
return ArtifactTypeUnknown
|
||||||
}
|
}
|
||||||
func (d *defaultProcessor) ListAdditionTypes() []string {
|
func (d *defaultProcessor) ListAdditionTypes(ctx context.Context, artifact *artifact.Artifact) []string {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func (d *defaultProcessor) AbstractMetadata(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error {
|
|
||||||
// do nothing currently
|
// The default processor will process user-defined artifact.
|
||||||
// we can extend this function to abstract the metadata in the future if needed
|
// AbstractMetadata will abstract data in a specific way.
|
||||||
|
// Annotation keys in artifact annotation will decide which content will be processed in artifact.
|
||||||
|
// Here is a manifest example:
|
||||||
|
// {
|
||||||
|
// "schemaVersion": 2,
|
||||||
|
// "config": {
|
||||||
|
// "mediaType": "application/vnd.caicloud.model.config.v1alpha1+json",
|
||||||
|
// "digest": "sha256:be948daf0e22f264ea70b713ea0db35050ae659c185706aa2fad74834455fe8c",
|
||||||
|
// "size": 187,
|
||||||
|
// "annotations": {
|
||||||
|
// "io.goharbor.artifact.v1alpha1.skip-list": "metrics,git"
|
||||||
|
// }
|
||||||
|
// },
|
||||||
|
// "layers": [
|
||||||
|
// {
|
||||||
|
// "mediaType": "image/png",
|
||||||
|
// "digest": "sha256:d923b93eadde0af5c639a972710a4d919066aba5d0dfbf4b9385099f70272da0",
|
||||||
|
// "size": 166015,
|
||||||
|
// "annotations": {
|
||||||
|
// "io.goharbor.artifact.v1alpha1.icon": ""
|
||||||
|
// }
|
||||||
|
// },
|
||||||
|
// {
|
||||||
|
// "mediaType": "application/tar+gzip",
|
||||||
|
// "digest": "sha256:d923b93eadde0af5c639a972710a4d919066aba5d0dfbf4b9385099f70272da0",
|
||||||
|
// "size": 166015
|
||||||
|
// }
|
||||||
|
// ]
|
||||||
|
// }
|
||||||
|
func (d *defaultProcessor) AbstractMetadata(ctx context.Context, artifact *artifact.Artifact, manifest []byte) error {
|
||||||
|
if artifact.ManifestMediaType != v1.MediaTypeImageManifest && artifact.ManifestMediaType != schema2.MediaTypeManifest {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// get manifest
|
||||||
|
mani := &v1.Manifest{}
|
||||||
|
if err := json.Unmarshal(manifest, mani); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// get config layer
|
||||||
|
_, blob, err := d.regCli.PullBlob(artifact.RepositoryName, mani.Config.Digest.String())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer blob.Close()
|
||||||
|
// parse metadata from config layer
|
||||||
|
metadata := map[string]interface{}{}
|
||||||
|
// Some artifact may not have empty config layer.
|
||||||
|
if mani.Config.Size != 0 {
|
||||||
|
if err := json.NewDecoder(blob).Decode(&metadata); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Populate all metadata into the ExtraAttrs first.
|
||||||
|
artifact.ExtraAttrs = metadata
|
||||||
|
annotationParser := annotation.NewParser()
|
||||||
|
err = annotationParser.Parse(ctx, artifact, manifest)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("the annotation parser parse annotation for artifact error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if artifact.Icon == "" {
|
||||||
|
artifact.Icon = DefaultIconDigest
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func (d *defaultProcessor) AbstractAddition(ctx context.Context, artifact *artifact.Artifact, addition string) (*Addition, error) {
|
func (d *defaultProcessor) AbstractAddition(ctx context.Context, artifact *artifact.Artifact, addition string) (*Addition, error) {
|
||||||
|
// Addition not support for user-defined artifact yet.
|
||||||
|
// It will be support in the future.
|
||||||
// return error directly
|
// return error directly
|
||||||
return nil, errors.New(nil).WithCode(errors.BadRequestCode).
|
return nil, errors.New(nil).WithCode(errors.BadRequestCode).
|
||||||
WithMessage("the processor for artifact %s not found, cannot get the addition", artifact.Type)
|
WithMessage("the processor for artifact %s not found, cannot get the addition", artifact.Type)
|
||||||
|
@ -15,39 +15,175 @@
|
|||||||
package processor
|
package processor
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/stretchr/testify/suite"
|
"context"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/distribution"
|
||||||
|
"github.com/goharbor/harbor/src/testing/mock"
|
||||||
|
v1 "github.com/opencontainers/image-spec/specs-go/v1"
|
||||||
|
"io/ioutil"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/artifact"
|
||||||
|
"github.com/goharbor/harbor/src/testing/pkg/parser"
|
||||||
|
"github.com/goharbor/harbor/src/testing/pkg/registry"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ormbConfig = `{
|
||||||
|
"created": "2015-10-31T22:22:56.015925234Z",
|
||||||
|
"author": "Ce Gao <gaoce@caicloud.io>",
|
||||||
|
"description": "CNN Model",
|
||||||
|
"tags": [
|
||||||
|
"cv"
|
||||||
|
],
|
||||||
|
"labels": {
|
||||||
|
"tensorflow.version": "2.0.0"
|
||||||
|
},
|
||||||
|
"framework": "TensorFlow",
|
||||||
|
"format": "SavedModel",
|
||||||
|
"size": 9223372036854775807,
|
||||||
|
"metrics": [
|
||||||
|
{
|
||||||
|
"name": "acc",
|
||||||
|
"value": "0.9"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"hyperparameters": [
|
||||||
|
{
|
||||||
|
"name": "batch_size",
|
||||||
|
"value": "32"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"signature": {
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"name": "input_1",
|
||||||
|
"size": [
|
||||||
|
224,
|
||||||
|
224,
|
||||||
|
3
|
||||||
|
],
|
||||||
|
"dtype": "float64"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "output_1",
|
||||||
|
"size": [
|
||||||
|
1,
|
||||||
|
1000
|
||||||
|
],
|
||||||
|
"dtype": "float64"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"layers": [
|
||||||
|
{
|
||||||
|
"name": "conv"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"training": {
|
||||||
|
"git": {
|
||||||
|
"repository": "git@github.com:caicloud/ormb.git",
|
||||||
|
"revision": "22f1d8406d464b0c0874075539c1f2e96c253775"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"dataset": {
|
||||||
|
"git": {
|
||||||
|
"repository": "git@github.com:caicloud/ormb.git",
|
||||||
|
"revision": "22f1d8406d464b0c0874075539c1f2e96c253775"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}`
|
||||||
|
ormbManifestWithoutIcon = `{
|
||||||
|
"schemaVersion":2,
|
||||||
|
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
||||||
|
"config":{
|
||||||
|
"mediaType":"application/vnd.caicloud.model.config.v1alpha1+json",
|
||||||
|
"digest":"sha256:be948daf0e22f264ea70b713ea0db35050ae659c185706aa2fad74834455fe8c",
|
||||||
|
"size":187,
|
||||||
|
"annotations": {
|
||||||
|
"io.goharbor.artifact.v1alpha1.skip-list": "metrics,git"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"layers":[
|
||||||
|
{
|
||||||
|
"mediaType":"application/tar+gzip",
|
||||||
|
"digest":"sha256:eb6063fecbb50a9d98268cb61746a0fd62a27a4af9e850ffa543a1a62d3948b2",
|
||||||
|
"size":166022
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`
|
||||||
)
|
)
|
||||||
|
|
||||||
type defaultProcessorTestSuite struct {
|
type defaultProcessorTestSuite struct {
|
||||||
suite.Suite
|
suite.Suite
|
||||||
|
processor *defaultProcessor
|
||||||
|
parser *parser.Parser
|
||||||
|
regCli *registry.FakeClient
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *defaultProcessorTestSuite) SetupTest() {
|
||||||
|
d.regCli = ®istry.FakeClient{}
|
||||||
|
d.processor = &defaultProcessor{
|
||||||
|
regCli: d.regCli,
|
||||||
|
}
|
||||||
|
d.parser = &parser.Parser{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *defaultProcessorTestSuite) TestGetArtifactType() {
|
func (d *defaultProcessorTestSuite) TestGetArtifactType() {
|
||||||
mediaType := ""
|
mediaType := ""
|
||||||
processor := &defaultProcessor{mediaType: mediaType}
|
art := &artifact.Artifact{MediaType: mediaType}
|
||||||
typee := processor.GetArtifactType()
|
processor := &defaultProcessor{}
|
||||||
|
typee := processor.GetArtifactType(nil, art)
|
||||||
d.Equal(ArtifactTypeUnknown, typee)
|
d.Equal(ArtifactTypeUnknown, typee)
|
||||||
|
|
||||||
mediaType = "unknown"
|
mediaType = "unknown"
|
||||||
processor = &defaultProcessor{mediaType: mediaType}
|
art = &artifact.Artifact{MediaType: mediaType}
|
||||||
typee = processor.GetArtifactType()
|
processor = &defaultProcessor{}
|
||||||
|
typee = processor.GetArtifactType(nil, art)
|
||||||
d.Equal(ArtifactTypeUnknown, typee)
|
d.Equal(ArtifactTypeUnknown, typee)
|
||||||
|
|
||||||
mediaType = "application/vnd.oci.image.config.v1+json"
|
mediaType = "application/vnd.oci.image.config.v1+json"
|
||||||
processor = &defaultProcessor{mediaType: mediaType}
|
art = &artifact.Artifact{MediaType: mediaType}
|
||||||
typee = processor.GetArtifactType()
|
processor = &defaultProcessor{}
|
||||||
|
typee = processor.GetArtifactType(nil, art)
|
||||||
d.Equal("IMAGE", typee)
|
d.Equal("IMAGE", typee)
|
||||||
|
|
||||||
mediaType = "application/vnd.cncf.helm.chart.config.v1+json"
|
mediaType = "application/vnd.cncf.helm.chart.config.v1+json"
|
||||||
processor = &defaultProcessor{mediaType: mediaType}
|
art = &artifact.Artifact{MediaType: mediaType}
|
||||||
typee = processor.GetArtifactType()
|
processor = &defaultProcessor{}
|
||||||
|
typee = processor.GetArtifactType(nil, art)
|
||||||
d.Equal("HELM.CHART", typee)
|
d.Equal("HELM.CHART", typee)
|
||||||
|
|
||||||
mediaType = "application/vnd.sylabs.sif.config.v1+json"
|
mediaType = "application/vnd.sylabs.sif.config.v1+json"
|
||||||
processor = &defaultProcessor{mediaType: mediaType}
|
art = &artifact.Artifact{MediaType: mediaType}
|
||||||
typee = processor.GetArtifactType()
|
processor = &defaultProcessor{}
|
||||||
|
typee = processor.GetArtifactType(nil, art)
|
||||||
d.Equal("SIF", typee)
|
d.Equal("SIF", typee)
|
||||||
|
|
||||||
|
mediaType = "application/vnd.caicloud.model.config.v1alpha1+json"
|
||||||
|
art = &artifact.Artifact{MediaType: mediaType}
|
||||||
|
processor = &defaultProcessor{}
|
||||||
|
typee = processor.GetArtifactType(nil, art)
|
||||||
|
d.Equal("MODEL", typee)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *defaultProcessorTestSuite) TestAbstractMetadata() {
|
||||||
|
manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(ormbManifestWithoutIcon))
|
||||||
|
d.Require().Nil(err)
|
||||||
|
manifestMediaType, content, err := manifest.Payload()
|
||||||
|
d.Require().Nil(err)
|
||||||
|
|
||||||
|
configBlob := ioutil.NopCloser(strings.NewReader(ormbConfig))
|
||||||
|
art := &artifact.Artifact{ManifestMediaType: manifestMediaType}
|
||||||
|
d.regCli.On("PullBlob").Return(0, configBlob, nil)
|
||||||
|
d.parser.On("Parse", context.TODO(), mock.AnythingOfType("*artifact.Artifact"), mock.AnythingOfType("[]byte")).Return(nil)
|
||||||
|
err = d.processor.AbstractMetadata(nil, art, content)
|
||||||
|
d.Require().Nil(err)
|
||||||
|
d.Equal(DefaultIconDigest, art.Icon)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDefaultProcessorTestSuite(t *testing.T) {
|
func TestDefaultProcessorTestSuite(t *testing.T) {
|
||||||
|
@ -15,10 +15,12 @@
|
|||||||
package image
|
package image
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"github.com/docker/distribution/manifest/manifestlist"
|
"github.com/docker/distribution/manifest/manifestlist"
|
||||||
"github.com/goharbor/harbor/src/controller/artifact/processor"
|
"github.com/goharbor/harbor/src/controller/artifact/processor"
|
||||||
"github.com/goharbor/harbor/src/controller/artifact/processor/base"
|
"github.com/goharbor/harbor/src/controller/artifact/processor/base"
|
||||||
"github.com/goharbor/harbor/src/lib/log"
|
"github.com/goharbor/harbor/src/lib/log"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/artifact"
|
||||||
v1 "github.com/opencontainers/image-spec/specs-go/v1"
|
v1 "github.com/opencontainers/image-spec/specs-go/v1"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -40,6 +42,6 @@ type indexProcessor struct {
|
|||||||
*base.IndexProcessor
|
*base.IndexProcessor
|
||||||
}
|
}
|
||||||
|
|
||||||
func (i *indexProcessor) GetArtifactType() string {
|
func (i *indexProcessor) GetArtifactType(ctx context.Context, artifact *artifact.Artifact) string {
|
||||||
return ArtifactTypeImage
|
return ArtifactTypeImage
|
||||||
}
|
}
|
||||||
|
@ -29,7 +29,7 @@ func (i *indexProcessTestSuite) SetupTest() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (i *indexProcessTestSuite) TestGetArtifactType() {
|
func (i *indexProcessTestSuite) TestGetArtifactType() {
|
||||||
i.Assert().Equal(ArtifactTypeImage, i.processor.GetArtifactType())
|
i.Assert().Equal(ArtifactTypeImage, i.processor.GetArtifactType(nil, nil))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestIndexProcessTestSuite(t *testing.T) {
|
func TestIndexProcessTestSuite(t *testing.T) {
|
||||||
|
@ -36,7 +36,7 @@ func init() {
|
|||||||
type manifestV1Processor struct {
|
type manifestV1Processor struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *manifestV1Processor) AbstractMetadata(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error {
|
func (m *manifestV1Processor) AbstractMetadata(ctx context.Context, artifact *artifact.Artifact, manifest []byte) error {
|
||||||
mani := &schema1.Manifest{}
|
mani := &schema1.Manifest{}
|
||||||
if err := json.Unmarshal(manifest, mani); err != nil {
|
if err := json.Unmarshal(manifest, mani); err != nil {
|
||||||
return err
|
return err
|
||||||
@ -53,10 +53,10 @@ func (m *manifestV1Processor) AbstractAddition(ctx context.Context, artifact *ar
|
|||||||
WithMessage("addition %s isn't supported for %s(manifest version 1)", addition, ArtifactTypeImage)
|
WithMessage("addition %s isn't supported for %s(manifest version 1)", addition, ArtifactTypeImage)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *manifestV1Processor) GetArtifactType() string {
|
func (m *manifestV1Processor) GetArtifactType(ctx context.Context, artifact *artifact.Artifact) string {
|
||||||
return ArtifactTypeImage
|
return ArtifactTypeImage
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *manifestV1Processor) ListAdditionTypes() []string {
|
func (m *manifestV1Processor) ListAdditionTypes(ctx context.Context, artifact *artifact.Artifact) []string {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -78,7 +78,7 @@ func (m *manifestV1ProcessorTestSuite) TestAbstractMetadata() {
|
|||||||
}
|
}
|
||||||
`
|
`
|
||||||
artifact := &artifact.Artifact{}
|
artifact := &artifact.Artifact{}
|
||||||
err := m.processor.AbstractMetadata(nil, []byte(manifest), artifact)
|
err := m.processor.AbstractMetadata(nil, artifact, []byte(manifest))
|
||||||
m.Require().Nil(err)
|
m.Require().Nil(err)
|
||||||
m.Assert().Equal("amd64", artifact.ExtraAttrs["architecture"].(string))
|
m.Assert().Equal("amd64", artifact.ExtraAttrs["architecture"].(string))
|
||||||
}
|
}
|
||||||
@ -89,11 +89,11 @@ func (m *manifestV1ProcessorTestSuite) TestAbstractAddition() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m *manifestV1ProcessorTestSuite) TestGetArtifactType() {
|
func (m *manifestV1ProcessorTestSuite) TestGetArtifactType() {
|
||||||
m.Assert().Equal(ArtifactTypeImage, m.processor.GetArtifactType())
|
m.Assert().Equal(ArtifactTypeImage, m.processor.GetArtifactType(nil, nil))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *manifestV1ProcessorTestSuite) TestListAdditionTypes() {
|
func (m *manifestV1ProcessorTestSuite) TestListAdditionTypes() {
|
||||||
additions := m.processor.ListAdditionTypes()
|
additions := m.processor.ListAdditionTypes(nil, nil)
|
||||||
m.Len(additions, 0)
|
m.Len(additions, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -86,10 +86,10 @@ func (m *manifestV2Processor) AbstractAddition(ctx context.Context, artifact *ar
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *manifestV2Processor) GetArtifactType() string {
|
func (m *manifestV2Processor) GetArtifactType(ctx context.Context, artifact *artifact.Artifact) string {
|
||||||
return ArtifactTypeImage
|
return ArtifactTypeImage
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *manifestV2Processor) ListAdditionTypes() []string {
|
func (m *manifestV2Processor) ListAdditionTypes(ctx context.Context, artifact *artifact.Artifact) []string {
|
||||||
return []string{AdditionTypeBuildHistory}
|
return []string{AdditionTypeBuildHistory}
|
||||||
}
|
}
|
||||||
|
@ -153,11 +153,11 @@ func (m *manifestV2ProcessorTestSuite) TestAbstractAddition() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m *manifestV2ProcessorTestSuite) TestGetArtifactType() {
|
func (m *manifestV2ProcessorTestSuite) TestGetArtifactType() {
|
||||||
m.Assert().Equal(ArtifactTypeImage, m.processor.GetArtifactType())
|
m.Assert().Equal(ArtifactTypeImage, m.processor.GetArtifactType(nil, nil))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *manifestV2ProcessorTestSuite) TestListAdditionTypes() {
|
func (m *manifestV2ProcessorTestSuite) TestListAdditionTypes() {
|
||||||
additions := m.processor.ListAdditionTypes()
|
additions := m.processor.ListAdditionTypes(nil, nil)
|
||||||
m.EqualValues([]string{AdditionTypeBuildHistory}, additions)
|
m.EqualValues([]string{AdditionTypeBuildHistory}, additions)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -17,6 +17,7 @@ package processor
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/lib/log"
|
"github.com/goharbor/harbor/src/lib/log"
|
||||||
"github.com/goharbor/harbor/src/pkg/artifact"
|
"github.com/goharbor/harbor/src/pkg/artifact"
|
||||||
)
|
)
|
||||||
@ -35,12 +36,12 @@ type Addition struct {
|
|||||||
// Processor processes specified artifact
|
// Processor processes specified artifact
|
||||||
type Processor interface {
|
type Processor interface {
|
||||||
// GetArtifactType returns the type of one kind of artifact specified by media type
|
// GetArtifactType returns the type of one kind of artifact specified by media type
|
||||||
GetArtifactType() string
|
GetArtifactType(ctx context.Context, artifact *artifact.Artifact) string
|
||||||
// ListAdditionTypes returns the supported addition types of one kind of artifact specified by media type
|
// ListAdditionTypes returns the supported addition types of one kind of artifact specified by media type
|
||||||
ListAdditionTypes() []string
|
ListAdditionTypes(ctx context.Context, artifact *artifact.Artifact) []string
|
||||||
// AbstractMetadata abstracts the metadata for the specific artifact type into the artifact model,
|
// AbstractMetadata abstracts the metadata for the specific artifact type into the artifact model,
|
||||||
// the metadata can be got from the manifest or other layers referenced by the manifest.
|
// the metadata can be got from the manifest or other layers referenced by the manifest.
|
||||||
AbstractMetadata(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error
|
AbstractMetadata(ctx context.Context, artifact *artifact.Artifact, manifest []byte) error
|
||||||
// AbstractAddition abstracts the addition of the artifact.
|
// AbstractAddition abstracts the addition of the artifact.
|
||||||
// The additions are different for different artifacts:
|
// The additions are different for different artifacts:
|
||||||
// build history for image; values.yaml, readme and dependencies for chart, etc
|
// build history for image; values.yaml, readme and dependencies for chart, etc
|
||||||
@ -66,7 +67,7 @@ func Get(mediaType string) Processor {
|
|||||||
// no registered processor found, use the default one
|
// no registered processor found, use the default one
|
||||||
if processor == nil {
|
if processor == nil {
|
||||||
log.Debugf("the processor for media type %s not found, use the default one", mediaType)
|
log.Debugf("the processor for media type %s not found, use the default one", mediaType)
|
||||||
processor = &defaultProcessor{mediaType: mediaType}
|
processor = DefaultProcessor
|
||||||
}
|
}
|
||||||
return processor
|
return processor
|
||||||
}
|
}
|
||||||
|
@ -23,13 +23,13 @@ import (
|
|||||||
|
|
||||||
type fakeProcessor struct{}
|
type fakeProcessor struct{}
|
||||||
|
|
||||||
func (f *fakeProcessor) GetArtifactType() string {
|
func (f *fakeProcessor) GetArtifactType(ctx context.Context, artifact *artifact.Artifact) string {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
func (f *fakeProcessor) ListAdditionTypes() []string {
|
func (f *fakeProcessor) ListAdditionTypes(ctx context.Context, artifact *artifact.Artifact) []string {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func (f *fakeProcessor) AbstractMetadata(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error {
|
func (f *fakeProcessor) AbstractMetadata(ctx context.Context, artifact *artifact.Artifact, manifest []byte) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func (f *fakeProcessor) AbstractAddition(ctx context.Context, artifact *artifact.Artifact, additionType string) (*Addition, error) {
|
func (f *fakeProcessor) AbstractAddition(ctx context.Context, artifact *artifact.Artifact, additionType string) (*Addition, error) {
|
||||||
|
@ -40,6 +40,7 @@ type Artifact struct {
|
|||||||
PullTime time.Time `orm:"column(pull_time)"`
|
PullTime time.Time `orm:"column(pull_time)"`
|
||||||
ExtraAttrs string `orm:"column(extra_attrs)"` // json string
|
ExtraAttrs string `orm:"column(extra_attrs)"` // json string
|
||||||
Annotations string `orm:"column(annotations);type(jsonb)"` // json string
|
Annotations string `orm:"column(annotations);type(jsonb)"` // json string
|
||||||
|
Icon string `orm:"column(icon)"` // icon layer digest
|
||||||
}
|
}
|
||||||
|
|
||||||
// TableName for artifact
|
// TableName for artifact
|
||||||
|
@ -43,6 +43,7 @@ type Artifact struct {
|
|||||||
ExtraAttrs map[string]interface{} `json:"extra_attrs"` // only contains the simple attributes specific for the different artifact type, most of them should come from the config layer
|
ExtraAttrs map[string]interface{} `json:"extra_attrs"` // only contains the simple attributes specific for the different artifact type, most of them should come from the config layer
|
||||||
Annotations map[string]string `json:"annotations"`
|
Annotations map[string]string `json:"annotations"`
|
||||||
References []*Reference `json:"references"` // child artifacts referenced by the parent artifact if the artifact is an index
|
References []*Reference `json:"references"` // child artifacts referenced by the parent artifact if the artifact is an index
|
||||||
|
Icon string `json:"icon"` // icon layer digest
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsImageIndex returns true when artifact is image index
|
// IsImageIndex returns true when artifact is image index
|
||||||
|
30
src/testing/pkg/parser/parser.go
Normal file
30
src/testing/pkg/parser/parser.go
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
// Code generated by mockery v2.1.0. DO NOT EDIT.
|
||||||
|
|
||||||
|
package parser
|
||||||
|
|
||||||
|
import (
|
||||||
|
context "context"
|
||||||
|
|
||||||
|
artifact "github.com/goharbor/harbor/src/pkg/artifact"
|
||||||
|
|
||||||
|
mock "github.com/stretchr/testify/mock"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Parser is an autogenerated mock type for the Parser type
|
||||||
|
type Parser struct {
|
||||||
|
mock.Mock
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse provides a mock function with given fields: ctx, _a1, manifest
|
||||||
|
func (_m *Parser) Parse(ctx context.Context, _a1 *artifact.Artifact, manifest []byte) error {
|
||||||
|
ret := _m.Called(ctx, _a1, manifest)
|
||||||
|
|
||||||
|
var r0 error
|
||||||
|
if rf, ok := ret.Get(0).(func(context.Context, *artifact.Artifact, []byte) error); ok {
|
||||||
|
r0 = rf(ctx, _a1, manifest)
|
||||||
|
} else {
|
||||||
|
r0 = ret.Error(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
return r0
|
||||||
|
}
|
85
src/testing/pkg/processor/processor.go
Normal file
85
src/testing/pkg/processor/processor.go
Normal file
@ -0,0 +1,85 @@
|
|||||||
|
// Code generated by mockery v2.1.0. DO NOT EDIT.
|
||||||
|
|
||||||
|
package processor
|
||||||
|
|
||||||
|
import (
|
||||||
|
context "context"
|
||||||
|
|
||||||
|
artifact "github.com/goharbor/harbor/src/pkg/artifact"
|
||||||
|
|
||||||
|
mock "github.com/stretchr/testify/mock"
|
||||||
|
|
||||||
|
processor "github.com/goharbor/harbor/src/controller/artifact/processor"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Processor is an autogenerated mock type for the Processor type
|
||||||
|
type Processor struct {
|
||||||
|
mock.Mock
|
||||||
|
}
|
||||||
|
|
||||||
|
// AbstractAddition provides a mock function with given fields: ctx, _a1, additionType
|
||||||
|
func (_m *Processor) AbstractAddition(ctx context.Context, _a1 *artifact.Artifact, additionType string) (*processor.Addition, error) {
|
||||||
|
ret := _m.Called(ctx, _a1, additionType)
|
||||||
|
|
||||||
|
var r0 *processor.Addition
|
||||||
|
if rf, ok := ret.Get(0).(func(context.Context, *artifact.Artifact, string) *processor.Addition); ok {
|
||||||
|
r0 = rf(ctx, _a1, additionType)
|
||||||
|
} else {
|
||||||
|
if ret.Get(0) != nil {
|
||||||
|
r0 = ret.Get(0).(*processor.Addition)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var r1 error
|
||||||
|
if rf, ok := ret.Get(1).(func(context.Context, *artifact.Artifact, string) error); ok {
|
||||||
|
r1 = rf(ctx, _a1, additionType)
|
||||||
|
} else {
|
||||||
|
r1 = ret.Error(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return r0, r1
|
||||||
|
}
|
||||||
|
|
||||||
|
// AbstractMetadata provides a mock function with given fields: ctx, _a1, manifest
|
||||||
|
func (_m *Processor) AbstractMetadata(ctx context.Context, _a1 *artifact.Artifact, manifest []byte) error {
|
||||||
|
ret := _m.Called(ctx, _a1, manifest)
|
||||||
|
|
||||||
|
var r0 error
|
||||||
|
if rf, ok := ret.Get(0).(func(context.Context, *artifact.Artifact, []byte) error); ok {
|
||||||
|
r0 = rf(ctx, _a1, manifest)
|
||||||
|
} else {
|
||||||
|
r0 = ret.Error(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
return r0
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetArtifactType provides a mock function with given fields: ctx, _a1
|
||||||
|
func (_m *Processor) GetArtifactType(ctx context.Context, _a1 *artifact.Artifact) string {
|
||||||
|
ret := _m.Called(ctx, _a1)
|
||||||
|
|
||||||
|
var r0 string
|
||||||
|
if rf, ok := ret.Get(0).(func(context.Context, *artifact.Artifact) string); ok {
|
||||||
|
r0 = rf(ctx, _a1)
|
||||||
|
} else {
|
||||||
|
r0 = ret.Get(0).(string)
|
||||||
|
}
|
||||||
|
|
||||||
|
return r0
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListAdditionTypes provides a mock function with given fields: ctx, _a1
|
||||||
|
func (_m *Processor) ListAdditionTypes(ctx context.Context, _a1 *artifact.Artifact) []string {
|
||||||
|
ret := _m.Called(ctx, _a1)
|
||||||
|
|
||||||
|
var r0 []string
|
||||||
|
if rf, ok := ret.Get(0).(func(context.Context, *artifact.Artifact) []string); ok {
|
||||||
|
r0 = rf(ctx, _a1)
|
||||||
|
} else {
|
||||||
|
if ret.Get(0) != nil {
|
||||||
|
r0 = ret.Get(0).([]string)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return r0
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user