Merge pull request #10379 from ywk253100/191230_abstractor_pr

Implement the artifact abstractor and resolver
This commit is contained in:
Wenkai Yin(尹文开) 2020-01-13 10:23:41 +08:00 committed by GitHub
commit 3fe7517432
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 1023 additions and 26 deletions

View File

@ -0,0 +1,118 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package abstractor
import (
"context"
"encoding/json"
"fmt"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/api/artifact/abstractor/blob"
"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/repository"
"github.com/opencontainers/image-spec/specs-go/v1"
)
// Abstractor abstracts the specific information for different types of artifacts
type Abstractor interface {
// Abstract the specific information for the specific artifact type into the artifact model,
// the information can be got from the manifest or other layers referenced by the manifest.
Abstract(ctx context.Context, artifact *artifact.Artifact) error
}
// NewAbstractor returns an instance of the default abstractor
func NewAbstractor() Abstractor {
return &abstractor{
repoMgr: repository.Mgr,
blobFetcher: blob.Fcher,
}
}
type abstractor struct {
repoMgr repository.Manager
blobFetcher blob.Fetcher
}
// TODO try CNAB, how to forbid CNAB
// TODO add white list for supported artifact type
func (a *abstractor) Abstract(ctx context.Context, artifact *artifact.Artifact) error {
repository, err := a.repoMgr.Get(ctx, artifact.RepositoryID)
if err != nil {
return err
}
// read manifest content
manifestMediaType, content, err := a.blobFetcher.FetchManifest(repository.Name, artifact.Digest)
if err != nil {
return err
}
artifact.ManifestMediaType = manifestMediaType
switch artifact.ManifestMediaType {
// docker manifest v1
case "", "application/json", schema1.MediaTypeSignedManifest:
// TODO as the manifestmediatype isn't null, so add not null constraint in database
// unify the media type of v1 manifest to "schema1.MediaTypeSignedManifest"
artifact.ManifestMediaType = schema1.MediaTypeSignedManifest
// as no config layer in the docker v1 manifest, use the "schema1.MediaTypeSignedManifest"
// as the media type of artifact
artifact.MediaType = schema1.MediaTypeSignedManifest
// there is no layer size in v1 manifest, doesn't set the artifact size
// OCI manifest/docker manifest v2
case v1.MediaTypeImageManifest, schema2.MediaTypeManifest:
manifest := &v1.Manifest{}
if err := json.Unmarshal(content, manifest); err != nil {
return err
}
// use the "manifest.config.mediatype" as the media type of the artifact
artifact.MediaType = manifest.Config.MediaType
// set size
artifact.Size = int64(len(content)) + manifest.Config.Size
for _, layer := range manifest.Layers {
artifact.Size += layer.Size
}
// set annotations
artifact.Annotations = manifest.Annotations
// OCI index/docker manifest list
case v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList:
// the identity of index is still in progress, only handle image index for now
// and use the manifestMediaType as the media type of artifact
// If we want to support CNAB, we should get the media type from annotation
artifact.MediaType = artifact.ManifestMediaType
index := &v1.Index{}
if err := json.Unmarshal(content, index); err != nil {
return err
}
// the size for image index is meaningless, doesn't set it for image index
// but it is useful for CNAB or other artifacts, set it when needed
// set annotations
artifact.Annotations = index.Annotations
// TODO handle references in resolvers
default:
return fmt.Errorf("unsupported manifest media type: %s", artifact.ManifestMediaType)
}
resolver, err := resolver.Get(artifact.MediaType)
if err != nil {
return err
}
artifact.Type = resolver.ArtifactType()
return resolver.Resolve(ctx, content, artifact)
}

View File

@ -0,0 +1,299 @@
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package abstractor
import (
"context"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/artifact"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/goharbor/harbor/src/testing/api/artifact/abstractor/blob"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/stretchr/testify/suite"
"testing"
)
var (
fakeArtifactType = "FAKE_ARTIFACT"
v1Manifest = `{
"name": "hello-world",
"tag": "latest",
"architecture": "amd64",
"fsLayers": [
{
"blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"
},
{
"blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"
},
{
"blobSum": "sha256:cc8567d70002e957612902a8e985ea129d831ebe04057d88fb644857caa45d11"
},
{
"blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"
}
],
"history": [
{
"v1Compatibility": "{\"id\":\"e45a5af57b00862e5ef5782a9925979a02ba2b12dff832fd0991335f4a11e5c5\",\"parent\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"created\":\"2014-12-31T22:57:59.178729048Z\",\"container\":\"27b45f8fb11795b52e9605b686159729b0d9ca92f76d40fb4f05a62e19c46b4f\",\"container_config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [/hello]\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"docker_version\":\"1.4.1\",\"config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/hello\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
},
{
"v1Compatibility": "{\"id\":\"e45a5af57b00862e5ef5782a9925979a02ba2b12dff832fd0991335f4a11e5c5\",\"parent\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"created\":\"2014-12-31T22:57:59.178729048Z\",\"container\":\"27b45f8fb11795b52e9605b686159729b0d9ca92f76d40fb4f05a62e19c46b4f\",\"container_config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [/hello]\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"docker_version\":\"1.4.1\",\"config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/hello\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
},
],
"schemaVersion": 1,
"signatures": [
{
"header": {
"jwk": {
"crv": "P-256",
"kid": "OD6I:6DRK:JXEJ:KBM4:255X:NSAA:MUSF:E4VM:ZI6W:CUN2:L4Z6:LSF4",
"kty": "EC",
"x": "3gAwX48IQ5oaYQAYSxor6rYYc_6yjuLCjtQ9LUakg4A",
"y": "t72ge6kIA1XOjqjVoEOiPPAURltJFBMGDSQvEGVB010"
},
"alg": "ES256"
},
"signature": "XREm0L8WNn27Ga_iE_vRnTxVMhhYY0Zst_FfkKopg6gWSoTOZTuW4rK0fg_IqnKkEKlbD83tD46LKEGi5aIVFg",
"protected": "eyJmb3JtYXRMZW5ndGgiOjY2MjgsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wNC0wOFQxODo1Mjo1OVoifQ"
}
]
}`
v2Manifest = `{
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"config": {
"mediaType": "application/vnd.docker.container.image.v1+json",
"size": 1510,
"digest": "sha256:fce289e99eb9bca977dae136fbe2a82b6b7d4c372474c9235adc1741675f587e"
},
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": 977,
"digest": "sha256:1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced"
}
],
"annotations": {
"com.example.key1": "value1"
}
}`
v2Config = `{
"architecture": "amd64",
"config": {
"Hostname": "",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": [
"/hello"
],
"ArgsEscaped": true,
"Image": "sha256:a6d1aaad8ca65655449a26146699fe9d61240071f6992975be7e720f1cd42440",
"Volumes": null,
"WorkingDir": "",
"Entrypoint": null,
"OnBuild": null,
"Labels": null
},
"container": "8e2caa5a514bb6d8b4f2a2553e9067498d261a0fd83a96aeaaf303943dff6ff9",
"container_config": {
"Hostname": "8e2caa5a514b",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": [
"/bin/sh",
"-c",
"#(nop) ",
"CMD [\"/hello\"]"
],
"ArgsEscaped": true,
"Image": "sha256:a6d1aaad8ca65655449a26146699fe9d61240071f6992975be7e720f1cd42440",
"Volumes": null,
"WorkingDir": "",
"Entrypoint": null,
"OnBuild": null,
"Labels": {
}
},
"created": "2019-01-01T01:29:27.650294696Z",
"docker_version": "18.06.1-ce",
"history": [
{
"created": "2019-01-01T01:29:27.416803627Z",
"created_by": "/bin/sh -c #(nop) COPY file:f77490f70ce51da25bd21bfc30cb5e1a24b2b65eb37d4af0c327ddc24f0986a6 in / "
},
{
"created": "2019-01-01T01:29:27.650294696Z",
"created_by": "/bin/sh -c #(nop) CMD [\"/hello\"]",
"empty_layer": true
}
],
"os": "linux",
"rootfs": {
"type": "layers",
"diff_ids": [
"sha256:af0b15c8625bb1938f1d7b17081031f649fd14e6b233688eea3c5483994a66a3"
]
}
}`
index = `{
"schemaVersion": 2,
"manifests": [
{
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"size": 7143,
"digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
"platform": {
"architecture": "ppc64le",
"os": "linux"
}
},
{
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"size": 7682,
"digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
"platform": {
"architecture": "amd64",
"os": "linux"
}
}
],
"annotations": {
"com.example.key1": "value1"
}
}`
)
type fakeResolver struct{}
func (f *fakeResolver) ArtifactType() string {
return fakeArtifactType
}
func (f *fakeResolver) Resolve(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error {
return nil
}
type abstractorTestSuite struct {
suite.Suite
abstractor Abstractor
fetcher *blob.FakeFetcher
repoMgr *htesting.FakeRepositoryManager
}
func (a *abstractorTestSuite) SetupSuite() {
resolver.Register(&fakeResolver{}, schema1.MediaTypeSignedManifest,
schema2.MediaTypeImageConfig, v1.MediaTypeImageIndex)
}
func (a *abstractorTestSuite) SetupTest() {
a.fetcher = &blob.FakeFetcher{}
a.repoMgr = &htesting.FakeRepositoryManager{}
a.abstractor = &abstractor{
repoMgr: a.repoMgr,
blobFetcher: a.fetcher,
}
}
// docker manifest v1
func (a *abstractorTestSuite) TestAbstractV1Manifest() {
a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
a.fetcher.On("FetchManifest").Return(schema1.MediaTypeSignedManifest, []byte(v1Manifest), nil)
artifact := &artifact.Artifact{
ID: 1,
}
err := a.abstractor.Abstract(nil, artifact)
a.Require().Nil(err)
a.repoMgr.AssertExpectations(a.T())
a.fetcher.AssertExpectations(a.T())
a.Assert().Equal(int64(1), artifact.ID)
a.Assert().Equal(fakeArtifactType, artifact.Type)
a.Assert().Equal(schema1.MediaTypeSignedManifest, artifact.ManifestMediaType)
a.Assert().Equal(schema1.MediaTypeSignedManifest, artifact.MediaType)
a.Assert().Equal(int64(0), artifact.Size)
}
// docker manifest v2
func (a *abstractorTestSuite) TestAbstractV2Manifest() {
a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
a.fetcher.On("FetchManifest").Return(schema2.MediaTypeManifest, []byte(v2Manifest), nil)
artifact := &artifact.Artifact{
ID: 1,
}
err := a.abstractor.Abstract(nil, artifact)
a.Require().Nil(err)
a.repoMgr.AssertExpectations(a.T())
a.fetcher.AssertExpectations(a.T())
a.Assert().Equal(int64(1), artifact.ID)
a.Assert().Equal(fakeArtifactType, artifact.Type)
a.Assert().Equal(schema2.MediaTypeManifest, artifact.ManifestMediaType)
a.Assert().Equal(schema2.MediaTypeImageConfig, artifact.MediaType)
a.Assert().Equal(int64(3043), artifact.Size)
}
// OCI index
func (a *abstractorTestSuite) TestAbstractIndex() {
a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
a.fetcher.On("FetchManifest").Return(v1.MediaTypeImageIndex, []byte(index), nil)
artifact := &artifact.Artifact{
ID: 1,
}
err := a.abstractor.Abstract(nil, artifact)
a.Require().Nil(err)
a.repoMgr.AssertExpectations(a.T())
a.fetcher.AssertExpectations(a.T())
a.Assert().Equal(int64(1), artifact.ID)
a.Assert().Equal(fakeArtifactType, artifact.Type)
a.Assert().Equal(v1.MediaTypeImageIndex, artifact.ManifestMediaType)
a.Assert().Equal(v1.MediaTypeImageIndex, artifact.MediaType)
a.Assert().Equal(int64(0), artifact.Size)
a.Assert().Equal("value1", artifact.Annotations["com.example.key1"])
}
// OCI index
func (a *abstractorTestSuite) TestAbstractUnsupported() {
a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
a.fetcher.On("FetchManifest").Return("unsupported-manifest", []byte{}, nil)
artifact := &artifact.Artifact{
ID: 1,
}
err := a.abstractor.Abstract(nil, artifact)
a.Require().NotNil(err)
a.repoMgr.AssertExpectations(a.T())
a.fetcher.AssertExpectations(a.T())
}
func TestAbstractorTestSuite(t *testing.T) {
suite.Run(t, &abstractorTestSuite{})
}

View File

@ -0,0 +1,18 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
// TODO add cache
// TODO cache content and mediatype

View File

@ -0,0 +1,78 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
import (
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
coreutils "github.com/goharbor/harbor/src/core/utils"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"io/ioutil"
)
var (
// Fcher is a global blob fetcher instance
Fcher = NewFetcher()
accept = []string{
schema1.MediaTypeSignedManifest,
schema2.MediaTypeManifest,
v1.MediaTypeImageManifest,
manifestlist.MediaTypeManifestList,
v1.MediaTypeImageIndex,
}
)
// Fetcher fetches the content of blob
type Fetcher interface {
// FetchManifest the content of manifest under the repository
FetchManifest(repository, digest string) (mediaType string, content []byte, err error)
// FetchLayer the content of layer under the repository
FetchLayer(repository, digest string) (content []byte, err error)
}
// NewFetcher returns an instance of the default blob fetcher
func NewFetcher() Fetcher {
return &fetcher{}
}
type fetcher struct{}
// TODO re-implement it based on OCI registry driver
func (f *fetcher) FetchManifest(repository, digest string) (string, []byte, error) {
// TODO read from cache first
client, err := coreutils.NewRepositoryClientForLocal("admin", repository)
if err != nil {
return "", nil, err
}
_, mediaType, payload, err := client.PullManifest(digest, accept)
return mediaType, payload, err
}
// TODO re-implement it based on OCI registry driver
func (f *fetcher) FetchLayer(repository, digest string) ([]byte, error) {
// TODO read from cache first
client, err := coreutils.NewRepositoryClientForLocal("admin", repository)
if err != nil {
return nil, err
}
_, reader, err := client.PullBlob(digest)
if err != nil {
return nil, err
}
defer reader.Close()
return ioutil.ReadAll(reader)
}

View File

@ -0,0 +1,11 @@
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package chart

View File

@ -0,0 +1,42 @@
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package image
import (
"context"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/artifact"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
)
func init() {
rslver := &indexResolver{}
if err := resolver.Register(rslver, v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList); err != nil {
log.Errorf("failed to register resolver for artifact %s: %v", rslver.ArtifactType(), err)
return
}
}
// indexResolver resolves artifact with OCI index and docker manifest list
type indexResolver struct {
}
func (i *indexResolver) ArtifactType() string {
return ArtifactTypeImage
}
func (i *indexResolver) Resolve(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error {
// TODO implement
// how to make sure the artifact referenced by the index has already been saved in database
return nil
}

View File

@ -0,0 +1,40 @@
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package image
import (
"context"
"github.com/docker/distribution/manifest/schema1"
"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/artifact"
)
func init() {
rslver := &manifestV1Resolver{}
if err := resolver.Register(rslver, schema1.MediaTypeSignedManifest); err != nil {
log.Errorf("failed to register resolver for artifact %s: %v", rslver.ArtifactType(), err)
return
}
}
// manifestV1Resolver resolve artifact with docker v1 manifest
type manifestV1Resolver struct {
}
func (m *manifestV1Resolver) ArtifactType() string {
return ArtifactTypeImage
}
func (m *manifestV1Resolver) Resolve(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error {
// TODO implement
return nil
}

View File

@ -0,0 +1,76 @@
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package image
import (
"context"
"encoding/json"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/api/artifact/abstractor/blob"
"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/repository"
"github.com/opencontainers/image-spec/specs-go/v1"
)
const (
// ArtifactTypeImage is the artifact type for image
ArtifactTypeImage = "IMAGE"
)
func init() {
rslver := &manifestV2Resolver{
repoMgr: repository.Mgr,
blobFetcher: blob.Fcher,
}
if err := resolver.Register(rslver, v1.MediaTypeImageConfig, schema2.MediaTypeImageConfig); err != nil {
log.Errorf("failed to register resolver for artifact %s: %v", rslver.ArtifactType(), err)
return
}
}
// manifestV2Resolver resolve artifact with OCI manifest and docker v2 manifest
type manifestV2Resolver struct {
repoMgr repository.Manager
blobFetcher blob.Fetcher
}
func (m *manifestV2Resolver) ArtifactType() string {
return ArtifactTypeImage
}
func (m *manifestV2Resolver) Resolve(ctx context.Context, content []byte, artifact *artifact.Artifact) error {
repository, err := m.repoMgr.Get(ctx, artifact.RepositoryID)
if err != nil {
return err
}
manifest := &v1.Manifest{}
if err := json.Unmarshal(content, manifest); err != nil {
return err
}
digest := manifest.Config.Digest.String()
layer, err := m.blobFetcher.FetchLayer(repository.Name, digest)
if err != nil {
return err
}
image := &v1.Image{}
if err := json.Unmarshal(layer, image); err != nil {
return err
}
artifact.ExtraAttrs = map[string]interface{}{
"created": image.Created,
"author": image.Author,
"architecture": image.Architecture,
"os": image.OS,
}
return nil
}

View File

@ -0,0 +1,150 @@
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package image
import (
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/artifact"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/goharbor/harbor/src/testing/api/artifact/abstractor/blob"
"github.com/stretchr/testify/suite"
"testing"
)
type manifestV2ResolverTestSuite struct {
suite.Suite
resolver *manifestV2Resolver
repoMgr *htesting.FakeRepositoryManager
blobFetcher *blob.FakeFetcher
}
func (m *manifestV2ResolverTestSuite) SetupTest() {
m.repoMgr = &htesting.FakeRepositoryManager{}
m.blobFetcher = &blob.FakeFetcher{}
m.resolver = &manifestV2Resolver{
repoMgr: m.repoMgr,
blobFetcher: m.blobFetcher,
}
}
func (m *manifestV2ResolverTestSuite) TestArtifactType() {
m.Assert().Equal(ArtifactTypeImage, m.resolver.ArtifactType())
}
func (m *manifestV2ResolverTestSuite) TestResolve() {
content := `{
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"config": {
"mediaType": "application/vnd.docker.container.image.v1+json",
"size": 1510,
"digest": "sha256:fce289e99eb9bca977dae136fbe2a82b6b7d4c372474c9235adc1741675f587e"
},
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": 977,
"digest": "sha256:1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced"
}
]
}`
config := `{
"architecture": "amd64",
"config": {
"Hostname": "",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": [
"/hello"
],
"ArgsEscaped": true,
"Image": "sha256:a6d1aaad8ca65655449a26146699fe9d61240071f6992975be7e720f1cd42440",
"Volumes": null,
"WorkingDir": "",
"Entrypoint": null,
"OnBuild": null,
"Labels": null
},
"container": "8e2caa5a514bb6d8b4f2a2553e9067498d261a0fd83a96aeaaf303943dff6ff9",
"container_config": {
"Hostname": "8e2caa5a514b",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": [
"/bin/sh",
"-c",
"#(nop) ",
"CMD [\"/hello\"]"
],
"ArgsEscaped": true,
"Image": "sha256:a6d1aaad8ca65655449a26146699fe9d61240071f6992975be7e720f1cd42440",
"Volumes": null,
"WorkingDir": "",
"Entrypoint": null,
"OnBuild": null,
"Labels": {
}
},
"created": "2019-01-01T01:29:27.650294696Z",
"docker_version": "18.06.1-ce",
"history": [
{
"created": "2019-01-01T01:29:27.416803627Z",
"created_by": "/bin/sh -c #(nop) COPY file:f77490f70ce51da25bd21bfc30cb5e1a24b2b65eb37d4af0c327ddc24f0986a6 in / "
},
{
"created": "2019-01-01T01:29:27.650294696Z",
"created_by": "/bin/sh -c #(nop) CMD [\"/hello\"]",
"empty_layer": true
}
],
"os": "linux",
"rootfs": {
"type": "layers",
"diff_ids": [
"sha256:af0b15c8625bb1938f1d7b17081031f649fd14e6b233688eea3c5483994a66a3"
]
}
}`
artifact := &artifact.Artifact{}
m.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
m.blobFetcher.On("FetchLayer").Return([]byte(config), nil)
err := m.resolver.Resolve(nil, []byte(content), artifact)
m.Require().Nil(err)
m.repoMgr.AssertExpectations(m.T())
m.blobFetcher.AssertExpectations(m.T())
m.Assert().Equal("amd64", artifact.ExtraAttrs["architecture"].(string))
m.Assert().Equal("linux", artifact.ExtraAttrs["os"].(string))
}
func TestManifestV2ResolverTestSuite(t *testing.T) {
suite.Run(t, &manifestV2ResolverTestSuite{})
}

View File

@ -0,0 +1,58 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package resolver
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/artifact"
)
var (
registry = map[string]Resolver{}
)
// Resolver resolves the detail information for a specific kind of artifact
type Resolver interface {
// ArtifactType returns the type of artifact that the resolver handles
ArtifactType() string
// Resolve receives the manifest content, resolves the detail information
// from the manifest or the layers referenced by the manifest, and populates
// the detail information into the artifact
Resolve(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error
}
// Register resolver, one resolver can handle multiple media types for one kind of artifact
func Register(resolver Resolver, mediaTypes ...string) error {
for _, mediaType := range mediaTypes {
_, exist := registry[mediaType]
if exist {
return fmt.Errorf("resolver to handle media type %s already exists", mediaType)
}
registry[mediaType] = resolver
log.Infof("resolver to handle media type %s registered", mediaType)
}
return nil
}
// Get the resolver according to the media type
func Get(mediaType string) (Resolver, error) {
resolver, exist := registry[mediaType]
if !exist {
return nil, fmt.Errorf("resolver resolves %s not found", mediaType)
}
return resolver, nil
}

View File

@ -0,0 +1,54 @@
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package resolver
import (
"github.com/stretchr/testify/suite"
"testing"
)
type resolverTestSuite struct {
suite.Suite
}
func (r *resolverTestSuite) SetupTest() {
registry = map[string]Resolver{}
}
func (r *resolverTestSuite) TestRegister() {
// registry a resolver
mediaType := "fake_media_type"
err := Register(nil, mediaType)
r.Assert().Nil(err)
// try to register a resolver for the existing media type
err = Register(nil, mediaType)
r.Assert().NotNil(err)
}
func (r *resolverTestSuite) TestGet() {
// registry a resolver
mediaType := "fake_media_type"
err := Register(nil, mediaType)
r.Assert().Nil(err)
// get the resolver
_, err = Get(mediaType)
r.Assert().Nil(err)
// get the not exist resolver
_, err = Get("not_existing_media_type")
r.Assert().NotNil(err)
}
func TestResolverTestSuite(t *testing.T) {
suite.Run(t, &resolverTestSuite{})
}

View File

@ -17,6 +17,9 @@ package artifact
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/api/artifact/abstractor"
// registry image resolvers
_ "github.com/goharbor/harbor/src/api/artifact/abstractor/resolver/image"
"github.com/goharbor/harbor/src/common/utils/log"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/artifact"
@ -29,7 +32,7 @@ import (
var (
// Ctl is a global artifact controller instance
Ctl = NewController(repository.Mgr, artifact.Mgr, tag.Mgr)
Ctl = NewController()
)
// Controller defines the operations related with artifacts and tags
@ -63,11 +66,12 @@ type Controller interface {
}
// NewController creates an instance of the default artifact controller
func NewController(repoMgr repository.Manager, artMgr artifact.Manager, tagMgr tag.Manager) Controller {
func NewController() Controller {
return &controller{
repoMgr: repoMgr,
artMgr: artMgr,
tagMgr: tagMgr,
repoMgr: repository.Mgr,
artMgr: artifact.Mgr,
tagMgr: tag.Mgr,
abstractor: abstractor.NewAbstractor(),
}
}
@ -76,9 +80,10 @@ func NewController(repoMgr repository.Manager, artMgr artifact.Manager, tagMgr t
// for artifacts and tags
type controller struct {
repoMgr repository.Manager
artMgr artifact.Manager
tagMgr tag.Manager
repoMgr repository.Manager
artMgr artifact.Manager
tagMgr tag.Manager
abstractor abstractor.Abstractor
}
func (c *controller) Ensure(ctx context.Context, repositoryID int64, digest string, tags ...string) (bool, int64, error) {
@ -123,7 +128,10 @@ func (c *controller) ensureArtifact(ctx context.Context, repositoryID int64, dig
PushTime: time.Now(),
}
// abstract the specific information for the artifact
c.abstract(ctx, artifact)
if err = c.abstractor.Abstract(ctx, artifact); err != nil {
return false, 0, err
}
// create it
id, err := c.artMgr.Create(ctx, artifact)
if err != nil {
@ -301,8 +309,3 @@ func (c *controller) assembleTag(ctx context.Context, tag *tm.Tag, option *TagOp
// TODO populate label, signature, immutable status for tag
return t
}
func (c *controller) abstract(ctx context.Context, artifact *artifact.Artifact) {
// TODO abstract the specific info for the artifact
// handler references
}

View File

@ -15,32 +15,46 @@
package artifact
import (
"context"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/tag/model/tag"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"testing"
"time"
)
type fakeAbstractor struct {
mock.Mock
}
func (f *fakeAbstractor) Abstract(ctx context.Context, artifact *artifact.Artifact) error {
args := f.Called()
return args.Error(0)
}
type controllerTestSuite struct {
suite.Suite
ctl *controller
repoMgr *htesting.FakeRepositoryManager
artMgr *htesting.FakeArtifactManager
tagMgr *htesting.FakeTagManager
ctl *controller
repoMgr *htesting.FakeRepositoryManager
artMgr *htesting.FakeArtifactManager
tagMgr *htesting.FakeTagManager
abstractor *fakeAbstractor
}
func (c *controllerTestSuite) SetupTest() {
c.repoMgr = &htesting.FakeRepositoryManager{}
c.artMgr = &htesting.FakeArtifactManager{}
c.tagMgr = &htesting.FakeTagManager{}
c.abstractor = &fakeAbstractor{}
c.ctl = &controller{
repoMgr: c.repoMgr,
artMgr: c.artMgr,
tagMgr: c.tagMgr,
repoMgr: c.repoMgr,
artMgr: c.artMgr,
tagMgr: c.tagMgr,
abstractor: c.abstractor,
}
}
@ -94,10 +108,6 @@ func (c *controllerTestSuite) TestAssembleArtifact() {
// TODO check other fields of option
}
func (c *controllerTestSuite) TestAbstract() {
// TODO add test case
}
func (c *controllerTestSuite) TestEnsureArtifact() {
digest := "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180"
@ -117,16 +127,18 @@ func (c *controllerTestSuite) TestEnsureArtifact() {
// reset the mock
c.SetupTest()
// the artifact doesn't exist
c.repoMgr.On("Get").Return(&models.RepoRecord{
ProjectID: 1,
}, nil)
// the artifact doesn't exist
c.artMgr.On("List").Return(1, []*artifact.Artifact{}, nil)
c.artMgr.On("Create").Return(1, nil)
c.abstractor.On("Abstract").Return(nil)
created, id, err = c.ctl.ensureArtifact(nil, 1, digest)
c.Require().Nil(err)
c.repoMgr.AssertExpectations(c.T())
c.artMgr.AssertExpectations(c.T())
c.abstractor.AssertExpectations(c.T())
c.True(created)
c.Equal(int64(1), id)
}
@ -184,11 +196,13 @@ func (c *controllerTestSuite) TestEnsure() {
c.artMgr.On("Create").Return(1, nil)
c.tagMgr.On("List").Return(1, []*tag.Tag{}, nil)
c.tagMgr.On("Create").Return(1, nil)
c.abstractor.On("Abstract").Return(nil)
_, id, err := c.ctl.Ensure(nil, 1, digest, "latest")
c.Require().Nil(err)
c.repoMgr.AssertExpectations(c.T())
c.artMgr.AssertExpectations(c.T())
c.tagMgr.AssertExpectations(c.T())
c.abstractor.AssertExpectations(c.T())
c.Equal(int64(1), id)
}

View File

@ -0,0 +1,36 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
import (
"github.com/stretchr/testify/mock"
)
// FakeFetcher is a fake blob fetcher that implement the src/api/artifact/blob.Fetcher interface
type FakeFetcher struct {
mock.Mock
}
// FetchManifest ...
func (f *FakeFetcher) FetchManifest(repoFullName, digest string) (string, []byte, error) {
args := f.Called(mock.Anything)
return args.String(0), args.Get(1).([]byte), args.Error(2)
}
// FetchLayer ...
func (f *FakeFetcher) FetchLayer(repoFullName, digest string) (content []byte, err error) {
args := f.Called(mock.Anything)
return args.Get(0).([]byte), args.Error(1)
}