Implement the artifact abstractor and resolver

1. Define the interface for artifact abstractor and resolver
2. Implement the artifact abstractor
3. Implement the resolver for image with manifest v2

Signed-off-by: Wenkai Yin <yinw@vmware.com>

src/api/artifact/abstractor/abstractor.go

@@ -0,0 +1,118 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package abstractor
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/goharbor/harbor/src/api/artifact/abstractor/blob"
+	"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
+	"github.com/goharbor/harbor/src/pkg/artifact"
+	"github.com/goharbor/harbor/src/pkg/repository"
+	"github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// Abstractor abstracts the specific information for different types of artifacts
+type Abstractor interface {
+	// Abstract the specific information for the specific artifact type into the artifact model,
+	// the information can be got from the manifest or other layers referenced by the manifest.
+	Abstract(ctx context.Context, artifact *artifact.Artifact) error
+}
+
+// NewAbstractor returns an instance of the default abstractor
+func NewAbstractor() Abstractor {
+	return &abstractor{
+		repoMgr:     repository.Mgr,
+		blobFetcher: blob.Fcher,
+	}
+}
+
+type abstractor struct {
+	repoMgr     repository.Manager
+	blobFetcher blob.Fetcher
+}
+
+// TODO try CNAB, how to forbid CNAB
+
+// TODO add white list for supported artifact type
+func (a *abstractor) Abstract(ctx context.Context, artifact *artifact.Artifact) error {
+	repository, err := a.repoMgr.Get(ctx, artifact.RepositoryID)
+	if err != nil {
+		return err
+	}
+	// read manifest content
+	manifestMediaType, content, err := a.blobFetcher.FetchManifest(repository.Name, artifact.Digest)
+	if err != nil {
+		return err
+	}
+	artifact.ManifestMediaType = manifestMediaType
+
+	switch artifact.ManifestMediaType {
+	// docker manifest v1
+	case "", "application/json", schema1.MediaTypeSignedManifest:
+		// TODO as the manifestmediatype isn't null, so add not null constraint in database
+		// unify the media type of v1 manifest to "schema1.MediaTypeSignedManifest"
+		artifact.ManifestMediaType = schema1.MediaTypeSignedManifest
+		// as no config layer in the docker v1 manifest, use the "schema1.MediaTypeSignedManifest"
+		// as the media type of artifact
+		artifact.MediaType = schema1.MediaTypeSignedManifest
+		// there is no layer size in v1 manifest, doesn't set the artifact size
+	// OCI manifest/docker manifest v2
+	case v1.MediaTypeImageManifest, schema2.MediaTypeManifest:
+		manifest := &v1.Manifest{}
+		if err := json.Unmarshal(content, manifest); err != nil {
+			return err
+		}
+		// use the "manifest.config.mediatype" as the media type of the artifact
+		artifact.MediaType = manifest.Config.MediaType
+		// set size
+		artifact.Size = int64(len(content)) + manifest.Config.Size
+		for _, layer := range manifest.Layers {
+			artifact.Size += layer.Size
+		}
+		// set annotations
+		artifact.Annotations = manifest.Annotations
+	// OCI index/docker manifest list
+	case v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList:
+		// the identity of index is still in progress, only handle image index for now
+		// and use the manifestMediaType as the media type of artifact
+		// If we want to support CNAB, we should get the media type from annotation
+		artifact.MediaType = artifact.ManifestMediaType
+
+		index := &v1.Index{}
+		if err := json.Unmarshal(content, index); err != nil {
+			return err
+		}
+		// the size for image index is meaningless, doesn't set it for image index
+		// but it is useful for CNAB or other artifacts, set it when needed
+
+		// set annotations
+		artifact.Annotations = index.Annotations
+		// TODO handle references in resolvers
+	default:
+		return fmt.Errorf("unsupported manifest media type: %s", artifact.ManifestMediaType)
+	}
+
+	resolver, err := resolver.Get(artifact.MediaType)
+	if err != nil {
+		return err
+	}
+	artifact.Type = resolver.ArtifactType()
+	return resolver.Resolve(ctx, content, artifact)
+}

src/api/artifact/abstractor/abstractor_test.go

@@ -0,0 +1,299 @@
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package abstractor
+
+import (
+	"context"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/pkg/artifact"
+	htesting "github.com/goharbor/harbor/src/testing"
+	"github.com/goharbor/harbor/src/testing/api/artifact/abstractor/blob"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/stretchr/testify/suite"
+	"testing"
+)
+
+var (
+	fakeArtifactType = "FAKE_ARTIFACT"
+	v1Manifest       = `{
+   "name": "hello-world",
+   "tag": "latest",
+   "architecture": "amd64",
+   "fsLayers": [
+      {
+         "blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"
+      },
+      {
+         "blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"
+      },
+      {
+         "blobSum": "sha256:cc8567d70002e957612902a8e985ea129d831ebe04057d88fb644857caa45d11"
+      },
+      {
+         "blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"
+      }
+   ],
+   "history": [
+      {
+         "v1Compatibility": "{\"id\":\"e45a5af57b00862e5ef5782a9925979a02ba2b12dff832fd0991335f4a11e5c5\",\"parent\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"created\":\"2014-12-31T22:57:59.178729048Z\",\"container\":\"27b45f8fb11795b52e9605b686159729b0d9ca92f76d40fb4f05a62e19c46b4f\",\"container_config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [/hello]\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"docker_version\":\"1.4.1\",\"config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/hello\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
+      },
+      {
+         "v1Compatibility": "{\"id\":\"e45a5af57b00862e5ef5782a9925979a02ba2b12dff832fd0991335f4a11e5c5\",\"parent\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"created\":\"2014-12-31T22:57:59.178729048Z\",\"container\":\"27b45f8fb11795b52e9605b686159729b0d9ca92f76d40fb4f05a62e19c46b4f\",\"container_config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [/hello]\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"docker_version\":\"1.4.1\",\"config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/hello\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
+      },
+   ],
+   "schemaVersion": 1,
+   "signatures": [
+      {
+         "header": {
+            "jwk": {
+               "crv": "P-256",
+               "kid": "OD6I:6DRK:JXEJ:KBM4:255X:NSAA:MUSF:E4VM:ZI6W:CUN2:L4Z6:LSF4",
+               "kty": "EC",
+               "x": "3gAwX48IQ5oaYQAYSxor6rYYc_6yjuLCjtQ9LUakg4A",
+               "y": "t72ge6kIA1XOjqjVoEOiPPAURltJFBMGDSQvEGVB010"
+            },
+            "alg": "ES256"
+         },
+         "signature": "XREm0L8WNn27Ga_iE_vRnTxVMhhYY0Zst_FfkKopg6gWSoTOZTuW4rK0fg_IqnKkEKlbD83tD46LKEGi5aIVFg",
+         "protected": "eyJmb3JtYXRMZW5ndGgiOjY2MjgsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wNC0wOFQxODo1Mjo1OVoifQ"
+      }
+   ]
+}`
+	v2Manifest = `{
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+  "config": {
+    "mediaType": "application/vnd.docker.container.image.v1+json",
+    "size": 1510,
+    "digest": "sha256:fce289e99eb9bca977dae136fbe2a82b6b7d4c372474c9235adc1741675f587e"
+  },
+  "layers": [
+    {
+      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+      "size": 977,
+      "digest": "sha256:1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced"
+    }
+  ],
+  "annotations": {
+    "com.example.key1": "value1"
+  }
+}`
+	v2Config = `{
+  "architecture": "amd64",
+  "config": {
+    "Hostname": "",
+    "Domainname": "",
+    "User": "",
+    "AttachStdin": false,
+    "AttachStdout": false,
+    "AttachStderr": false,
+    "Tty": false,
+    "OpenStdin": false,
+    "StdinOnce": false,
+    "Env": [
+      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    ],
+    "Cmd": [
+      "/hello"
+    ],
+    "ArgsEscaped": true,
+    "Image": "sha256:a6d1aaad8ca65655449a26146699fe9d61240071f6992975be7e720f1cd42440",
+    "Volumes": null,
+    "WorkingDir": "",
+    "Entrypoint": null,
+    "OnBuild": null,
+    "Labels": null
+  },
+  "container": "8e2caa5a514bb6d8b4f2a2553e9067498d261a0fd83a96aeaaf303943dff6ff9",
+  "container_config": {
+    "Hostname": "8e2caa5a514b",
+    "Domainname": "",
+    "User": "",
+    "AttachStdin": false,
+    "AttachStdout": false,
+    "AttachStderr": false,
+    "Tty": false,
+    "OpenStdin": false,
+    "StdinOnce": false,
+    "Env": [
+      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    ],
+    "Cmd": [
+      "/bin/sh",
+      "-c",
+      "#(nop) ",
+      "CMD [\"/hello\"]"
+    ],
+    "ArgsEscaped": true,
+    "Image": "sha256:a6d1aaad8ca65655449a26146699fe9d61240071f6992975be7e720f1cd42440",
+    "Volumes": null,
+    "WorkingDir": "",
+    "Entrypoint": null,
+    "OnBuild": null,
+    "Labels": {
+      
+    }
+  },
+  "created": "2019-01-01T01:29:27.650294696Z",
+  "docker_version": "18.06.1-ce",
+  "history": [
+    {
+      "created": "2019-01-01T01:29:27.416803627Z",
+      "created_by": "/bin/sh -c #(nop) COPY file:f77490f70ce51da25bd21bfc30cb5e1a24b2b65eb37d4af0c327ddc24f0986a6 in / "
+    },
+    {
+      "created": "2019-01-01T01:29:27.650294696Z",
+      "created_by": "/bin/sh -c #(nop)  CMD [\"/hello\"]",
+      "empty_layer": true
+    }
+  ],
+  "os": "linux",
+  "rootfs": {
+    "type": "layers",
+    "diff_ids": [
+      "sha256:af0b15c8625bb1938f1d7b17081031f649fd14e6b233688eea3c5483994a66a3"
+    ]
+  }
+}`
+	index = `{
+  "schemaVersion": 2,
+  "manifests": [
+    {
+      "mediaType": "application/vnd.oci.image.manifest.v1+json",
+      "size": 7143,
+      "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
+      "platform": {
+        "architecture": "ppc64le",
+        "os": "linux"
+      }
+    },
+    {
+      "mediaType": "application/vnd.oci.image.manifest.v1+json",
+      "size": 7682,
+      "digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
+      "platform": {
+        "architecture": "amd64",
+        "os": "linux"
+      }
+    }
+  ],
+  "annotations": {
+    "com.example.key1": "value1"
+  }
+}`
+)
+
+type fakeResolver struct{}
+
+func (f *fakeResolver) ArtifactType() string {
+	return fakeArtifactType
+
+}
+func (f *fakeResolver) Resolve(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error {
+	return nil
+}
+
+type abstractorTestSuite struct {
+	suite.Suite
+	abstractor Abstractor
+	fetcher    *blob.FakeFetcher
+	repoMgr    *htesting.FakeRepositoryManager
+}
+
+func (a *abstractorTestSuite) SetupSuite() {
+	resolver.Register(&fakeResolver{}, schema1.MediaTypeSignedManifest,
+		schema2.MediaTypeImageConfig, v1.MediaTypeImageIndex)
+}
+
+func (a *abstractorTestSuite) SetupTest() {
+	a.fetcher = &blob.FakeFetcher{}
+	a.repoMgr = &htesting.FakeRepositoryManager{}
+	a.abstractor = &abstractor{
+		repoMgr:     a.repoMgr,
+		blobFetcher: a.fetcher,
+	}
+}
+
+// docker manifest v1
+func (a *abstractorTestSuite) TestAbstractV1Manifest() {
+	a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
+	a.fetcher.On("FetchManifest").Return(schema1.MediaTypeSignedManifest, []byte(v1Manifest), nil)
+	artifact := &artifact.Artifact{
+		ID: 1,
+	}
+	err := a.abstractor.Abstract(nil, artifact)
+	a.Require().Nil(err)
+	a.repoMgr.AssertExpectations(a.T())
+	a.fetcher.AssertExpectations(a.T())
+	a.Assert().Equal(int64(1), artifact.ID)
+	a.Assert().Equal(fakeArtifactType, artifact.Type)
+	a.Assert().Equal(schema1.MediaTypeSignedManifest, artifact.ManifestMediaType)
+	a.Assert().Equal(schema1.MediaTypeSignedManifest, artifact.MediaType)
+	a.Assert().Equal(int64(0), artifact.Size)
+}
+
+// docker manifest v2
+func (a *abstractorTestSuite) TestAbstractV2Manifest() {
+	a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
+	a.fetcher.On("FetchManifest").Return(schema2.MediaTypeManifest, []byte(v2Manifest), nil)
+	artifact := &artifact.Artifact{
+		ID: 1,
+	}
+
+	err := a.abstractor.Abstract(nil, artifact)
+	a.Require().Nil(err)
+	a.repoMgr.AssertExpectations(a.T())
+	a.fetcher.AssertExpectations(a.T())
+	a.Assert().Equal(int64(1), artifact.ID)
+	a.Assert().Equal(fakeArtifactType, artifact.Type)
+	a.Assert().Equal(schema2.MediaTypeManifest, artifact.ManifestMediaType)
+	a.Assert().Equal(schema2.MediaTypeImageConfig, artifact.MediaType)
+	a.Assert().Equal(int64(3043), artifact.Size)
+}
+
+// OCI index
+func (a *abstractorTestSuite) TestAbstractIndex() {
+	a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
+	a.fetcher.On("FetchManifest").Return(v1.MediaTypeImageIndex, []byte(index), nil)
+	artifact := &artifact.Artifact{
+		ID: 1,
+	}
+	err := a.abstractor.Abstract(nil, artifact)
+	a.Require().Nil(err)
+	a.repoMgr.AssertExpectations(a.T())
+	a.fetcher.AssertExpectations(a.T())
+	a.Assert().Equal(int64(1), artifact.ID)
+	a.Assert().Equal(fakeArtifactType, artifact.Type)
+	a.Assert().Equal(v1.MediaTypeImageIndex, artifact.ManifestMediaType)
+	a.Assert().Equal(v1.MediaTypeImageIndex, artifact.MediaType)
+	a.Assert().Equal(int64(0), artifact.Size)
+	a.Assert().Equal("value1", artifact.Annotations["com.example.key1"])
+}
+
+// OCI index
+func (a *abstractorTestSuite) TestAbstractUnsupported() {
+	a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
+	a.fetcher.On("FetchManifest").Return("unsupported-manifest", []byte{}, nil)
+	artifact := &artifact.Artifact{
+		ID: 1,
+	}
+	err := a.abstractor.Abstract(nil, artifact)
+	a.Require().NotNil(err)
+	a.repoMgr.AssertExpectations(a.T())
+	a.fetcher.AssertExpectations(a.T())
+}
+
+func TestAbstractorTestSuite(t *testing.T) {
+	suite.Run(t, &abstractorTestSuite{})
+}

src/api/artifact/abstractor/blob/cache.go

@@ -0,0 +1,18 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package blob
+
+// TODO add cache
+// TODO cache content and mediatype

src/api/artifact/abstractor/blob/fetcher.go

@@ -0,0 +1,78 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package blob
+
+import (
+	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/manifest/schema2"
+	coreutils "github.com/goharbor/harbor/src/core/utils"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"io/ioutil"
+)
+
+var (
+	// Fcher is a global blob fetcher instance
+	Fcher = NewFetcher()
+
+	accept = []string{
+		schema1.MediaTypeSignedManifest,
+		schema2.MediaTypeManifest,
+		v1.MediaTypeImageManifest,
+		manifestlist.MediaTypeManifestList,
+		v1.MediaTypeImageIndex,
+	}
+)
+
+// Fetcher fetches the content of blob
+type Fetcher interface {
+	// FetchManifest the content of manifest under the repository
+	FetchManifest(repository, digest string) (mediaType string, content []byte, err error)
+	// FetchLayer the content of layer under the repository
+	FetchLayer(repository, digest string) (content []byte, err error)
+}
+
+// NewFetcher returns an instance of the default blob fetcher
+func NewFetcher() Fetcher {
+	return &fetcher{}
+}
+
+type fetcher struct{}
+
+// TODO re-implement it based on OCI registry driver
+func (f *fetcher) FetchManifest(repository, digest string) (string, []byte, error) {
+	// TODO read from cache first
+	client, err := coreutils.NewRepositoryClientForLocal("admin", repository)
+	if err != nil {
+		return "", nil, err
+	}
+	_, mediaType, payload, err := client.PullManifest(digest, accept)
+	return mediaType, payload, err
+}
+
+// TODO re-implement it based on OCI registry driver
+func (f *fetcher) FetchLayer(repository, digest string) ([]byte, error) {
+	// TODO read from cache first
+	client, err := coreutils.NewRepositoryClientForLocal("admin", repository)
+	if err != nil {
+		return nil, err
+	}
+	_, reader, err := client.PullBlob(digest)
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+	return ioutil.ReadAll(reader)
+}

src/api/artifact/abstractor/resolver/chart/chart.go

@@ -0,0 +1,11 @@
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chart

src/api/artifact/abstractor/resolver/image/index.go

@@ -0,0 +1,42 @@
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package image
+
+import (
+	"context"
+	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/pkg/artifact"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+func init() {
+	rslver := &indexResolver{}
+	if err := resolver.Register(rslver, v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList); err != nil {
+		log.Errorf("failed to register resolver for artifact %s: %v", rslver.ArtifactType(), err)
+		return
+	}
+}
+
+// indexResolver resolves artifact with OCI index and docker manifest list
+type indexResolver struct {
+}
+
+func (i *indexResolver) ArtifactType() string {
+	return ArtifactTypeImage
+}
+
+func (i *indexResolver) Resolve(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error {
+	// TODO implement
+	// how to make sure the artifact referenced by the index has already been saved in database
+	return nil
+}

src/api/artifact/abstractor/resolver/image/manifest_v1.go

@@ -0,0 +1,40 @@
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package image
+
+import (
+	"context"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/pkg/artifact"
+)
+
+func init() {
+	rslver := &manifestV1Resolver{}
+	if err := resolver.Register(rslver, schema1.MediaTypeSignedManifest); err != nil {
+		log.Errorf("failed to register resolver for artifact %s: %v", rslver.ArtifactType(), err)
+		return
+	}
+}
+
+// manifestV1Resolver resolve artifact with docker v1 manifest
+type manifestV1Resolver struct {
+}
+
+func (m *manifestV1Resolver) ArtifactType() string {
+	return ArtifactTypeImage
+}
+
+func (m *manifestV1Resolver) Resolve(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error {
+	// TODO implement
+	return nil
+}

src/api/artifact/abstractor/resolver/image/manifest_v2.go

@@ -0,0 +1,76 @@
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package image
+
+import (
+	"context"
+	"encoding/json"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/goharbor/harbor/src/api/artifact/abstractor/blob"
+	"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/pkg/artifact"
+	"github.com/goharbor/harbor/src/pkg/repository"
+	"github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+const (
+	// ArtifactTypeImage is the artifact type for image
+	ArtifactTypeImage = "IMAGE"
+)
+
+func init() {
+	rslver := &manifestV2Resolver{
+		repoMgr:     repository.Mgr,
+		blobFetcher: blob.Fcher,
+	}
+	if err := resolver.Register(rslver, v1.MediaTypeImageConfig, schema2.MediaTypeImageConfig); err != nil {
+		log.Errorf("failed to register resolver for artifact %s: %v", rslver.ArtifactType(), err)
+		return
+	}
+}
+
+// manifestV2Resolver resolve artifact with OCI manifest and docker v2 manifest
+type manifestV2Resolver struct {
+	repoMgr     repository.Manager
+	blobFetcher blob.Fetcher
+}
+
+func (m *manifestV2Resolver) ArtifactType() string {
+	return ArtifactTypeImage
+}
+
+func (m *manifestV2Resolver) Resolve(ctx context.Context, content []byte, artifact *artifact.Artifact) error {
+	repository, err := m.repoMgr.Get(ctx, artifact.RepositoryID)
+	if err != nil {
+		return err
+	}
+	manifest := &v1.Manifest{}
+	if err := json.Unmarshal(content, manifest); err != nil {
+		return err
+	}
+	digest := manifest.Config.Digest.String()
+	layer, err := m.blobFetcher.FetchLayer(repository.Name, digest)
+	if err != nil {
+		return err
+	}
+	image := &v1.Image{}
+	if err := json.Unmarshal(layer, image); err != nil {
+		return err
+	}
+	artifact.ExtraAttrs = map[string]interface{}{
+		"created":      image.Created,
+		"author":       image.Author,
+		"architecture": image.Architecture,
+		"os":           image.OS,
+	}
+	return nil
+}

src/api/artifact/abstractor/resolver/image/manifest_v2_test.go

@@ -0,0 +1,150 @@
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package image
+
+import (
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/pkg/artifact"
+	htesting "github.com/goharbor/harbor/src/testing"
+	"github.com/goharbor/harbor/src/testing/api/artifact/abstractor/blob"
+	"github.com/stretchr/testify/suite"
+	"testing"
+)
+
+type manifestV2ResolverTestSuite struct {
+	suite.Suite
+	resolver    *manifestV2Resolver
+	repoMgr     *htesting.FakeRepositoryManager
+	blobFetcher *blob.FakeFetcher
+}
+
+func (m *manifestV2ResolverTestSuite) SetupTest() {
+	m.repoMgr = &htesting.FakeRepositoryManager{}
+	m.blobFetcher = &blob.FakeFetcher{}
+	m.resolver = &manifestV2Resolver{
+		repoMgr:     m.repoMgr,
+		blobFetcher: m.blobFetcher,
+	}
+
+}
+
+func (m *manifestV2ResolverTestSuite) TestArtifactType() {
+	m.Assert().Equal(ArtifactTypeImage, m.resolver.ArtifactType())
+}
+
+func (m *manifestV2ResolverTestSuite) TestResolve() {
+	content := `{
+   "schemaVersion": 2,
+   "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+   "config": {
+      "mediaType": "application/vnd.docker.container.image.v1+json",
+      "size": 1510,
+      "digest": "sha256:fce289e99eb9bca977dae136fbe2a82b6b7d4c372474c9235adc1741675f587e"
+   },
+   "layers": [
+      {
+         "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+         "size": 977,
+         "digest": "sha256:1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced"
+      }
+   ]
+}`
+	config := `{
+  "architecture": "amd64",
+  "config": {
+    "Hostname": "",
+    "Domainname": "",
+    "User": "",
+    "AttachStdin": false,
+    "AttachStdout": false,
+    "AttachStderr": false,
+    "Tty": false,
+    "OpenStdin": false,
+    "StdinOnce": false,
+    "Env": [
+      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    ],
+    "Cmd": [
+      "/hello"
+    ],
+    "ArgsEscaped": true,
+    "Image": "sha256:a6d1aaad8ca65655449a26146699fe9d61240071f6992975be7e720f1cd42440",
+    "Volumes": null,
+    "WorkingDir": "",
+    "Entrypoint": null,
+    "OnBuild": null,
+    "Labels": null
+  },
+  "container": "8e2caa5a514bb6d8b4f2a2553e9067498d261a0fd83a96aeaaf303943dff6ff9",
+  "container_config": {
+    "Hostname": "8e2caa5a514b",
+    "Domainname": "",
+    "User": "",
+    "AttachStdin": false,
+    "AttachStdout": false,
+    "AttachStderr": false,
+    "Tty": false,
+    "OpenStdin": false,
+    "StdinOnce": false,
+    "Env": [
+      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    ],
+    "Cmd": [
+      "/bin/sh",
+      "-c",
+      "#(nop) ",
+      "CMD [\"/hello\"]"
+    ],
+    "ArgsEscaped": true,
+    "Image": "sha256:a6d1aaad8ca65655449a26146699fe9d61240071f6992975be7e720f1cd42440",
+    "Volumes": null,
+    "WorkingDir": "",
+    "Entrypoint": null,
+    "OnBuild": null,
+    "Labels": {
+      
+    }
+  },
+  "created": "2019-01-01T01:29:27.650294696Z",
+  "docker_version": "18.06.1-ce",
+  "history": [
+    {
+      "created": "2019-01-01T01:29:27.416803627Z",
+      "created_by": "/bin/sh -c #(nop) COPY file:f77490f70ce51da25bd21bfc30cb5e1a24b2b65eb37d4af0c327ddc24f0986a6 in / "
+    },
+    {
+      "created": "2019-01-01T01:29:27.650294696Z",
+      "created_by": "/bin/sh -c #(nop)  CMD [\"/hello\"]",
+      "empty_layer": true
+    }
+  ],
+  "os": "linux",
+  "rootfs": {
+    "type": "layers",
+    "diff_ids": [
+      "sha256:af0b15c8625bb1938f1d7b17081031f649fd14e6b233688eea3c5483994a66a3"
+    ]
+  }
+}`
+	artifact := &artifact.Artifact{}
+	m.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
+	m.blobFetcher.On("FetchLayer").Return([]byte(config), nil)
+	err := m.resolver.Resolve(nil, []byte(content), artifact)
+	m.Require().Nil(err)
+	m.repoMgr.AssertExpectations(m.T())
+	m.blobFetcher.AssertExpectations(m.T())
+	m.Assert().Equal("amd64", artifact.ExtraAttrs["architecture"].(string))
+	m.Assert().Equal("linux", artifact.ExtraAttrs["os"].(string))
+}
+
+func TestManifestV2ResolverTestSuite(t *testing.T) {
+	suite.Run(t, &manifestV2ResolverTestSuite{})
+}

src/api/artifact/abstractor/resolver/resolver.go

@@ -0,0 +1,58 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package resolver
+
+import (
+	"context"
+	"fmt"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/pkg/artifact"
+)
+
+var (
+	registry = map[string]Resolver{}
+)
+
+// Resolver resolves the detail information for a specific kind of artifact
+type Resolver interface {
+	// ArtifactType returns the type of artifact that the resolver handles
+	ArtifactType() string
+	// Resolve receives the manifest content, resolves the detail information
+	// from the manifest or the layers referenced by the manifest, and populates
+	// the detail information into the artifact
+	Resolve(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error
+}
+
+// Register resolver, one resolver can handle multiple media types for one kind of artifact
+func Register(resolver Resolver, mediaTypes ...string) error {
+	for _, mediaType := range mediaTypes {
+		_, exist := registry[mediaType]
+		if exist {
+			return fmt.Errorf("resolver to handle media type %s already exists", mediaType)
+		}
+		registry[mediaType] = resolver
+		log.Infof("resolver to handle media type %s registered", mediaType)
+	}
+	return nil
+}
+
+// Get the resolver according to the media type
+func Get(mediaType string) (Resolver, error) {
+	resolver, exist := registry[mediaType]
+	if !exist {
+		return nil, fmt.Errorf("resolver resolves %s not found", mediaType)
+	}
+	return resolver, nil
+}

src/api/artifact/abstractor/resolver/resolver_test.go

@@ -0,0 +1,54 @@
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package resolver
+
+import (
+	"github.com/stretchr/testify/suite"
+	"testing"
+)
+
+type resolverTestSuite struct {
+	suite.Suite
+}
+
+func (r *resolverTestSuite) SetupTest() {
+	registry = map[string]Resolver{}
+}
+
+func (r *resolverTestSuite) TestRegister() {
+	// registry a resolver
+	mediaType := "fake_media_type"
+	err := Register(nil, mediaType)
+	r.Assert().Nil(err)
+
+	// try to register a resolver for the existing media type
+	err = Register(nil, mediaType)
+	r.Assert().NotNil(err)
+}
+
+func (r *resolverTestSuite) TestGet() {
+	// registry a resolver
+	mediaType := "fake_media_type"
+	err := Register(nil, mediaType)
+	r.Assert().Nil(err)
+
+	// get the resolver
+	_, err = Get(mediaType)
+	r.Assert().Nil(err)
+
+	// get the not exist resolver
+	_, err = Get("not_existing_media_type")
+	r.Assert().NotNil(err)
+}
+
+func TestResolverTestSuite(t *testing.T) {
+	suite.Run(t, &resolverTestSuite{})
+}

src/api/artifact/controller.go

@@ -17,6 +17,9 @@ package artifact
 import (
 	"context"
 	"fmt"
+	"github.com/goharbor/harbor/src/api/artifact/abstractor"
+	// registry image resolvers
+	_ "github.com/goharbor/harbor/src/api/artifact/abstractor/resolver/image"
 	"github.com/goharbor/harbor/src/common/utils/log"
 	ierror "github.com/goharbor/harbor/src/internal/error"
 	"github.com/goharbor/harbor/src/pkg/artifact"
@@ -29,7 +32,7 @@ import (
 
 var (
 	// Ctl is a global artifact controller instance
-	Ctl = NewController(repository.Mgr, artifact.Mgr, tag.Mgr)
+	Ctl = NewController()
 )
 
 // Controller defines the operations related with artifacts and tags
@@ -63,11 +66,12 @@ type Controller interface {
 }
 
 // NewController creates an instance of the default artifact controller
-func NewController(repoMgr repository.Manager, artMgr artifact.Manager, tagMgr tag.Manager) Controller {
+func NewController() Controller {
 	return &controller{
-		repoMgr: repoMgr,
+		repoMgr:    repository.Mgr,
-		artMgr:  artMgr,
+		artMgr:     artifact.Mgr,
-		tagMgr:  tagMgr,
+		tagMgr:     tag.Mgr,
+		abstractor: abstractor.NewAbstractor(),
 	}
 }
 
@@ -79,6 +83,7 @@ type controller struct {
 	repoMgr    repository.Manager
 	artMgr     artifact.Manager
 	tagMgr     tag.Manager
+	abstractor abstractor.Abstractor
 }
 
 func (c *controller) Ensure(ctx context.Context, repositoryID int64, digest string, tags ...string) (bool, int64, error) {
@@ -123,7 +128,10 @@ func (c *controller) ensureArtifact(ctx context.Context, repositoryID int64, dig
 		PushTime:     time.Now(),
 	}
 	// abstract the specific information for the artifact
-	c.abstract(ctx, artifact)
+	if err = c.abstractor.Abstract(ctx, artifact); err != nil {
+		return false, 0, err
+	}
+
 	// create it
 	id, err := c.artMgr.Create(ctx, artifact)
 	if err != nil {
@@ -301,8 +309,3 @@ func (c *controller) assembleTag(ctx context.Context, tag *tm.Tag, option *TagOp
 	// TODO populate label, signature, immutable status for tag
 	return t
 }
-
-func (c *controller) abstract(ctx context.Context, artifact *artifact.Artifact) {
-	// TODO abstract the specific info for the artifact
-	// handler references
-}

src/api/artifact/controller_test.go

@@ -15,32 +15,46 @@
 package artifact
 
 import (
+	"context"
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/pkg/artifact"
 	"github.com/goharbor/harbor/src/pkg/q"
 	"github.com/goharbor/harbor/src/pkg/tag/model/tag"
 	htesting "github.com/goharbor/harbor/src/testing"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/suite"
 	"testing"
 	"time"
 )
 
+type fakeAbstractor struct {
+	mock.Mock
+}
+
+func (f *fakeAbstractor) Abstract(ctx context.Context, artifact *artifact.Artifact) error {
+	args := f.Called()
+	return args.Error(0)
+}
+
 type controllerTestSuite struct {
 	suite.Suite
 	ctl        *controller
 	repoMgr    *htesting.FakeRepositoryManager
 	artMgr     *htesting.FakeArtifactManager
 	tagMgr     *htesting.FakeTagManager
+	abstractor *fakeAbstractor
 }
 
 func (c *controllerTestSuite) SetupTest() {
 	c.repoMgr = &htesting.FakeRepositoryManager{}
 	c.artMgr = &htesting.FakeArtifactManager{}
 	c.tagMgr = &htesting.FakeTagManager{}
+	c.abstractor = &fakeAbstractor{}
 	c.ctl = &controller{
 		repoMgr:    c.repoMgr,
 		artMgr:     c.artMgr,
 		tagMgr:     c.tagMgr,
+		abstractor: c.abstractor,
 	}
 }
 
@@ -94,10 +108,6 @@ func (c *controllerTestSuite) TestAssembleArtifact() {
 	// TODO check other fields of option
 }
 
-func (c *controllerTestSuite) TestAbstract() {
-	// TODO add test case
-}
-
 func (c *controllerTestSuite) TestEnsureArtifact() {
 	digest := "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180"
 
@@ -117,16 +127,18 @@ func (c *controllerTestSuite) TestEnsureArtifact() {
 	// reset the mock
 	c.SetupTest()
 
+	// the artifact doesn't exist
 	c.repoMgr.On("Get").Return(&models.RepoRecord{
 		ProjectID: 1,
 	}, nil)
-	// the artifact doesn't exist
 	c.artMgr.On("List").Return(1, []*artifact.Artifact{}, nil)
 	c.artMgr.On("Create").Return(1, nil)
+	c.abstractor.On("Abstract").Return(nil)
 	created, id, err = c.ctl.ensureArtifact(nil, 1, digest)
 	c.Require().Nil(err)
 	c.repoMgr.AssertExpectations(c.T())
 	c.artMgr.AssertExpectations(c.T())
+	c.abstractor.AssertExpectations(c.T())
 	c.True(created)
 	c.Equal(int64(1), id)
 }
@@ -184,11 +196,13 @@ func (c *controllerTestSuite) TestEnsure() {
 	c.artMgr.On("Create").Return(1, nil)
 	c.tagMgr.On("List").Return(1, []*tag.Tag{}, nil)
 	c.tagMgr.On("Create").Return(1, nil)
+	c.abstractor.On("Abstract").Return(nil)
 	_, id, err := c.ctl.Ensure(nil, 1, digest, "latest")
 	c.Require().Nil(err)
 	c.repoMgr.AssertExpectations(c.T())
 	c.artMgr.AssertExpectations(c.T())
 	c.tagMgr.AssertExpectations(c.T())
+	c.abstractor.AssertExpectations(c.T())
 	c.Equal(int64(1), id)
 }
 

src/testing/api/artifact/abstractor/blob/fetcher.go

@@ -0,0 +1,36 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package blob
+
+import (
+	"github.com/stretchr/testify/mock"
+)
+
+// FakeFetcher is a fake blob fetcher that implement the src/api/artifact/blob.Fetcher interface
+type FakeFetcher struct {
+	mock.Mock
+}
+
+// FetchManifest ...
+func (f *FakeFetcher) FetchManifest(repoFullName, digest string) (string, []byte, error) {
+	args := f.Called(mock.Anything)
+	return args.String(0), args.Get(1).([]byte), args.Error(2)
+}
+
+// FetchLayer ...
+func (f *FakeFetcher) FetchLayer(repoFullName, digest string) (content []byte, err error) {
+	args := f.Called(mock.Anything)
+	return args.Get(0).([]byte), args.Error(1)
+}