diff --git a/src/api/artifact/abstractor/abstractor.go b/src/api/artifact/abstractor/abstractor.go new file mode 100644 index 000000000..e1ea264e3 --- /dev/null +++ b/src/api/artifact/abstractor/abstractor.go @@ -0,0 +1,118 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package abstractor + +import ( + "context" + "encoding/json" + "fmt" + "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/distribution/manifest/schema2" + "github.com/goharbor/harbor/src/api/artifact/abstractor/blob" + "github.com/goharbor/harbor/src/api/artifact/abstractor/resolver" + "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/goharbor/harbor/src/pkg/repository" + "github.com/opencontainers/image-spec/specs-go/v1" +) + +// Abstractor abstracts the specific information for different types of artifacts +type Abstractor interface { + // Abstract the specific information for the specific artifact type into the artifact model, + // the information can be got from the manifest or other layers referenced by the manifest. + Abstract(ctx context.Context, artifact *artifact.Artifact) error +} + +// NewAbstractor returns an instance of the default abstractor +func NewAbstractor() Abstractor { + return &abstractor{ + repoMgr: repository.Mgr, + blobFetcher: blob.Fcher, + } +} + +type abstractor struct { + repoMgr repository.Manager + blobFetcher blob.Fetcher +} + +// TODO try CNAB, how to forbid CNAB + +// TODO add white list for supported artifact type +func (a *abstractor) Abstract(ctx context.Context, artifact *artifact.Artifact) error { + repository, err := a.repoMgr.Get(ctx, artifact.RepositoryID) + if err != nil { + return err + } + // read manifest content + manifestMediaType, content, err := a.blobFetcher.FetchManifest(repository.Name, artifact.Digest) + if err != nil { + return err + } + artifact.ManifestMediaType = manifestMediaType + + switch artifact.ManifestMediaType { + // docker manifest v1 + case "", "application/json", schema1.MediaTypeSignedManifest: + // TODO as the manifestmediatype isn't null, so add not null constraint in database + // unify the media type of v1 manifest to "schema1.MediaTypeSignedManifest" + artifact.ManifestMediaType = schema1.MediaTypeSignedManifest + // as no config layer in the docker v1 manifest, use the "schema1.MediaTypeSignedManifest" + // as the media type of artifact + artifact.MediaType = schema1.MediaTypeSignedManifest + // there is no layer size in v1 manifest, doesn't set the artifact size + // OCI manifest/docker manifest v2 + case v1.MediaTypeImageManifest, schema2.MediaTypeManifest: + manifest := &v1.Manifest{} + if err := json.Unmarshal(content, manifest); err != nil { + return err + } + // use the "manifest.config.mediatype" as the media type of the artifact + artifact.MediaType = manifest.Config.MediaType + // set size + artifact.Size = int64(len(content)) + manifest.Config.Size + for _, layer := range manifest.Layers { + artifact.Size += layer.Size + } + // set annotations + artifact.Annotations = manifest.Annotations + // OCI index/docker manifest list + case v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList: + // the identity of index is still in progress, only handle image index for now + // and use the manifestMediaType as the media type of artifact + // If we want to support CNAB, we should get the media type from annotation + artifact.MediaType = artifact.ManifestMediaType + + index := &v1.Index{} + if err := json.Unmarshal(content, index); err != nil { + return err + } + // the size for image index is meaningless, doesn't set it for image index + // but it is useful for CNAB or other artifacts, set it when needed + + // set annotations + artifact.Annotations = index.Annotations + // TODO handle references in resolvers + default: + return fmt.Errorf("unsupported manifest media type: %s", artifact.ManifestMediaType) + } + + resolver, err := resolver.Get(artifact.MediaType) + if err != nil { + return err + } + artifact.Type = resolver.ArtifactType() + return resolver.Resolve(ctx, content, artifact) +} diff --git a/src/api/artifact/abstractor/abstractor_test.go b/src/api/artifact/abstractor/abstractor_test.go new file mode 100644 index 000000000..0de9c609b --- /dev/null +++ b/src/api/artifact/abstractor/abstractor_test.go @@ -0,0 +1,299 @@ +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package abstractor + +import ( + "context" + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/distribution/manifest/schema2" + "github.com/goharbor/harbor/src/api/artifact/abstractor/resolver" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/pkg/artifact" + htesting "github.com/goharbor/harbor/src/testing" + "github.com/goharbor/harbor/src/testing/api/artifact/abstractor/blob" + v1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/suite" + "testing" +) + +var ( + fakeArtifactType = "FAKE_ARTIFACT" + v1Manifest = `{ + "name": "hello-world", + "tag": "latest", + "architecture": "amd64", + "fsLayers": [ + { + "blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" + }, + { + "blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" + }, + { + "blobSum": "sha256:cc8567d70002e957612902a8e985ea129d831ebe04057d88fb644857caa45d11" + }, + { + "blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" + } + ], + "history": [ + { + "v1Compatibility": "{\"id\":\"e45a5af57b00862e5ef5782a9925979a02ba2b12dff832fd0991335f4a11e5c5\",\"parent\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"created\":\"2014-12-31T22:57:59.178729048Z\",\"container\":\"27b45f8fb11795b52e9605b686159729b0d9ca92f76d40fb4f05a62e19c46b4f\",\"container_config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [/hello]\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"docker_version\":\"1.4.1\",\"config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/hello\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n" + }, + { + "v1Compatibility": "{\"id\":\"e45a5af57b00862e5ef5782a9925979a02ba2b12dff832fd0991335f4a11e5c5\",\"parent\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"created\":\"2014-12-31T22:57:59.178729048Z\",\"container\":\"27b45f8fb11795b52e9605b686159729b0d9ca92f76d40fb4f05a62e19c46b4f\",\"container_config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [/hello]\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"docker_version\":\"1.4.1\",\"config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/hello\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n" + }, + ], + "schemaVersion": 1, + "signatures": [ + { + "header": { + "jwk": { + "crv": "P-256", + "kid": "OD6I:6DRK:JXEJ:KBM4:255X:NSAA:MUSF:E4VM:ZI6W:CUN2:L4Z6:LSF4", + "kty": "EC", + "x": "3gAwX48IQ5oaYQAYSxor6rYYc_6yjuLCjtQ9LUakg4A", + "y": "t72ge6kIA1XOjqjVoEOiPPAURltJFBMGDSQvEGVB010" + }, + "alg": "ES256" + }, + "signature": "XREm0L8WNn27Ga_iE_vRnTxVMhhYY0Zst_FfkKopg6gWSoTOZTuW4rK0fg_IqnKkEKlbD83tD46LKEGi5aIVFg", + "protected": "eyJmb3JtYXRMZW5ndGgiOjY2MjgsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wNC0wOFQxODo1Mjo1OVoifQ" + } + ] +}` + v2Manifest = `{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 1510, + "digest": "sha256:fce289e99eb9bca977dae136fbe2a82b6b7d4c372474c9235adc1741675f587e" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 977, + "digest": "sha256:1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced" + } + ], + "annotations": { + "com.example.key1": "value1" + } +}` + v2Config = `{ + "architecture": "amd64", + "config": { + "Hostname": "", + "Domainname": "", + "User": "", + "AttachStdin": false, + "AttachStdout": false, + "AttachStderr": false, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ], + "Cmd": [ + "/hello" + ], + "ArgsEscaped": true, + "Image": "sha256:a6d1aaad8ca65655449a26146699fe9d61240071f6992975be7e720f1cd42440", + "Volumes": null, + "WorkingDir": "", + "Entrypoint": null, + "OnBuild": null, + "Labels": null + }, + "container": "8e2caa5a514bb6d8b4f2a2553e9067498d261a0fd83a96aeaaf303943dff6ff9", + "container_config": { + "Hostname": "8e2caa5a514b", + "Domainname": "", + "User": "", + "AttachStdin": false, + "AttachStdout": false, + "AttachStderr": false, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ], + "Cmd": [ + "/bin/sh", + "-c", + "#(nop) ", + "CMD [\"/hello\"]" + ], + "ArgsEscaped": true, + "Image": "sha256:a6d1aaad8ca65655449a26146699fe9d61240071f6992975be7e720f1cd42440", + "Volumes": null, + "WorkingDir": "", + "Entrypoint": null, + "OnBuild": null, + "Labels": { + + } + }, + "created": "2019-01-01T01:29:27.650294696Z", + "docker_version": "18.06.1-ce", + "history": [ + { + "created": "2019-01-01T01:29:27.416803627Z", + "created_by": "/bin/sh -c #(nop) COPY file:f77490f70ce51da25bd21bfc30cb5e1a24b2b65eb37d4af0c327ddc24f0986a6 in / " + }, + { + "created": "2019-01-01T01:29:27.650294696Z", + "created_by": "/bin/sh -c #(nop) CMD [\"/hello\"]", + "empty_layer": true + } + ], + "os": "linux", + "rootfs": { + "type": "layers", + "diff_ids": [ + "sha256:af0b15c8625bb1938f1d7b17081031f649fd14e6b233688eea3c5483994a66a3" + ] + } +}` + index = `{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7143, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + "platform": { + "architecture": "ppc64le", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7682, + "digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270", + "platform": { + "architecture": "amd64", + "os": "linux" + } + } + ], + "annotations": { + "com.example.key1": "value1" + } +}` +) + +type fakeResolver struct{} + +func (f *fakeResolver) ArtifactType() string { + return fakeArtifactType + +} +func (f *fakeResolver) Resolve(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error { + return nil +} + +type abstractorTestSuite struct { + suite.Suite + abstractor Abstractor + fetcher *blob.FakeFetcher + repoMgr *htesting.FakeRepositoryManager +} + +func (a *abstractorTestSuite) SetupSuite() { + resolver.Register(&fakeResolver{}, schema1.MediaTypeSignedManifest, + schema2.MediaTypeImageConfig, v1.MediaTypeImageIndex) +} + +func (a *abstractorTestSuite) SetupTest() { + a.fetcher = &blob.FakeFetcher{} + a.repoMgr = &htesting.FakeRepositoryManager{} + a.abstractor = &abstractor{ + repoMgr: a.repoMgr, + blobFetcher: a.fetcher, + } +} + +// docker manifest v1 +func (a *abstractorTestSuite) TestAbstractV1Manifest() { + a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil) + a.fetcher.On("FetchManifest").Return(schema1.MediaTypeSignedManifest, []byte(v1Manifest), nil) + artifact := &artifact.Artifact{ + ID: 1, + } + err := a.abstractor.Abstract(nil, artifact) + a.Require().Nil(err) + a.repoMgr.AssertExpectations(a.T()) + a.fetcher.AssertExpectations(a.T()) + a.Assert().Equal(int64(1), artifact.ID) + a.Assert().Equal(fakeArtifactType, artifact.Type) + a.Assert().Equal(schema1.MediaTypeSignedManifest, artifact.ManifestMediaType) + a.Assert().Equal(schema1.MediaTypeSignedManifest, artifact.MediaType) + a.Assert().Equal(int64(0), artifact.Size) +} + +// docker manifest v2 +func (a *abstractorTestSuite) TestAbstractV2Manifest() { + a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil) + a.fetcher.On("FetchManifest").Return(schema2.MediaTypeManifest, []byte(v2Manifest), nil) + artifact := &artifact.Artifact{ + ID: 1, + } + + err := a.abstractor.Abstract(nil, artifact) + a.Require().Nil(err) + a.repoMgr.AssertExpectations(a.T()) + a.fetcher.AssertExpectations(a.T()) + a.Assert().Equal(int64(1), artifact.ID) + a.Assert().Equal(fakeArtifactType, artifact.Type) + a.Assert().Equal(schema2.MediaTypeManifest, artifact.ManifestMediaType) + a.Assert().Equal(schema2.MediaTypeImageConfig, artifact.MediaType) + a.Assert().Equal(int64(3043), artifact.Size) +} + +// OCI index +func (a *abstractorTestSuite) TestAbstractIndex() { + a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil) + a.fetcher.On("FetchManifest").Return(v1.MediaTypeImageIndex, []byte(index), nil) + artifact := &artifact.Artifact{ + ID: 1, + } + err := a.abstractor.Abstract(nil, artifact) + a.Require().Nil(err) + a.repoMgr.AssertExpectations(a.T()) + a.fetcher.AssertExpectations(a.T()) + a.Assert().Equal(int64(1), artifact.ID) + a.Assert().Equal(fakeArtifactType, artifact.Type) + a.Assert().Equal(v1.MediaTypeImageIndex, artifact.ManifestMediaType) + a.Assert().Equal(v1.MediaTypeImageIndex, artifact.MediaType) + a.Assert().Equal(int64(0), artifact.Size) + a.Assert().Equal("value1", artifact.Annotations["com.example.key1"]) +} + +// OCI index +func (a *abstractorTestSuite) TestAbstractUnsupported() { + a.repoMgr.On("Get").Return(&models.RepoRecord{}, nil) + a.fetcher.On("FetchManifest").Return("unsupported-manifest", []byte{}, nil) + artifact := &artifact.Artifact{ + ID: 1, + } + err := a.abstractor.Abstract(nil, artifact) + a.Require().NotNil(err) + a.repoMgr.AssertExpectations(a.T()) + a.fetcher.AssertExpectations(a.T()) +} + +func TestAbstractorTestSuite(t *testing.T) { + suite.Run(t, &abstractorTestSuite{}) +} diff --git a/src/api/artifact/abstractor/blob/cache.go b/src/api/artifact/abstractor/blob/cache.go new file mode 100644 index 000000000..bb09aaee9 --- /dev/null +++ b/src/api/artifact/abstractor/blob/cache.go @@ -0,0 +1,18 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package blob + +// TODO add cache +// TODO cache content and mediatype diff --git a/src/api/artifact/abstractor/blob/fetcher.go b/src/api/artifact/abstractor/blob/fetcher.go new file mode 100644 index 000000000..3cb938534 --- /dev/null +++ b/src/api/artifact/abstractor/blob/fetcher.go @@ -0,0 +1,78 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package blob + +import ( + "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/distribution/manifest/schema2" + coreutils "github.com/goharbor/harbor/src/core/utils" + v1 "github.com/opencontainers/image-spec/specs-go/v1" + "io/ioutil" +) + +var ( + // Fcher is a global blob fetcher instance + Fcher = NewFetcher() + + accept = []string{ + schema1.MediaTypeSignedManifest, + schema2.MediaTypeManifest, + v1.MediaTypeImageManifest, + manifestlist.MediaTypeManifestList, + v1.MediaTypeImageIndex, + } +) + +// Fetcher fetches the content of blob +type Fetcher interface { + // FetchManifest the content of manifest under the repository + FetchManifest(repository, digest string) (mediaType string, content []byte, err error) + // FetchLayer the content of layer under the repository + FetchLayer(repository, digest string) (content []byte, err error) +} + +// NewFetcher returns an instance of the default blob fetcher +func NewFetcher() Fetcher { + return &fetcher{} +} + +type fetcher struct{} + +// TODO re-implement it based on OCI registry driver +func (f *fetcher) FetchManifest(repository, digest string) (string, []byte, error) { + // TODO read from cache first + client, err := coreutils.NewRepositoryClientForLocal("admin", repository) + if err != nil { + return "", nil, err + } + _, mediaType, payload, err := client.PullManifest(digest, accept) + return mediaType, payload, err +} + +// TODO re-implement it based on OCI registry driver +func (f *fetcher) FetchLayer(repository, digest string) ([]byte, error) { + // TODO read from cache first + client, err := coreutils.NewRepositoryClientForLocal("admin", repository) + if err != nil { + return nil, err + } + _, reader, err := client.PullBlob(digest) + if err != nil { + return nil, err + } + defer reader.Close() + return ioutil.ReadAll(reader) +} diff --git a/src/api/artifact/abstractor/resolver/chart/chart.go b/src/api/artifact/abstractor/resolver/chart/chart.go new file mode 100644 index 000000000..89ed387fd --- /dev/null +++ b/src/api/artifact/abstractor/resolver/chart/chart.go @@ -0,0 +1,11 @@ +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chart diff --git a/src/api/artifact/abstractor/resolver/image/index.go b/src/api/artifact/abstractor/resolver/image/index.go new file mode 100644 index 000000000..c1125c1c3 --- /dev/null +++ b/src/api/artifact/abstractor/resolver/image/index.go @@ -0,0 +1,42 @@ +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package image + +import ( + "context" + "github.com/docker/distribution/manifest/manifestlist" + "github.com/goharbor/harbor/src/api/artifact/abstractor/resolver" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/artifact" + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +func init() { + rslver := &indexResolver{} + if err := resolver.Register(rslver, v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList); err != nil { + log.Errorf("failed to register resolver for artifact %s: %v", rslver.ArtifactType(), err) + return + } +} + +// indexResolver resolves artifact with OCI index and docker manifest list +type indexResolver struct { +} + +func (i *indexResolver) ArtifactType() string { + return ArtifactTypeImage +} + +func (i *indexResolver) Resolve(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error { + // TODO implement + // how to make sure the artifact referenced by the index has already been saved in database + return nil +} diff --git a/src/api/artifact/abstractor/resolver/image/manifest_v1.go b/src/api/artifact/abstractor/resolver/image/manifest_v1.go new file mode 100644 index 000000000..1e25c2efa --- /dev/null +++ b/src/api/artifact/abstractor/resolver/image/manifest_v1.go @@ -0,0 +1,40 @@ +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package image + +import ( + "context" + "github.com/docker/distribution/manifest/schema1" + "github.com/goharbor/harbor/src/api/artifact/abstractor/resolver" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/artifact" +) + +func init() { + rslver := &manifestV1Resolver{} + if err := resolver.Register(rslver, schema1.MediaTypeSignedManifest); err != nil { + log.Errorf("failed to register resolver for artifact %s: %v", rslver.ArtifactType(), err) + return + } +} + +// manifestV1Resolver resolve artifact with docker v1 manifest +type manifestV1Resolver struct { +} + +func (m *manifestV1Resolver) ArtifactType() string { + return ArtifactTypeImage +} + +func (m *manifestV1Resolver) Resolve(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error { + // TODO implement + return nil +} diff --git a/src/api/artifact/abstractor/resolver/image/manifest_v2.go b/src/api/artifact/abstractor/resolver/image/manifest_v2.go new file mode 100644 index 000000000..7238cb82d --- /dev/null +++ b/src/api/artifact/abstractor/resolver/image/manifest_v2.go @@ -0,0 +1,76 @@ +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package image + +import ( + "context" + "encoding/json" + "github.com/docker/distribution/manifest/schema2" + "github.com/goharbor/harbor/src/api/artifact/abstractor/blob" + "github.com/goharbor/harbor/src/api/artifact/abstractor/resolver" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/goharbor/harbor/src/pkg/repository" + "github.com/opencontainers/image-spec/specs-go/v1" +) + +const ( + // ArtifactTypeImage is the artifact type for image + ArtifactTypeImage = "IMAGE" +) + +func init() { + rslver := &manifestV2Resolver{ + repoMgr: repository.Mgr, + blobFetcher: blob.Fcher, + } + if err := resolver.Register(rslver, v1.MediaTypeImageConfig, schema2.MediaTypeImageConfig); err != nil { + log.Errorf("failed to register resolver for artifact %s: %v", rslver.ArtifactType(), err) + return + } +} + +// manifestV2Resolver resolve artifact with OCI manifest and docker v2 manifest +type manifestV2Resolver struct { + repoMgr repository.Manager + blobFetcher blob.Fetcher +} + +func (m *manifestV2Resolver) ArtifactType() string { + return ArtifactTypeImage +} + +func (m *manifestV2Resolver) Resolve(ctx context.Context, content []byte, artifact *artifact.Artifact) error { + repository, err := m.repoMgr.Get(ctx, artifact.RepositoryID) + if err != nil { + return err + } + manifest := &v1.Manifest{} + if err := json.Unmarshal(content, manifest); err != nil { + return err + } + digest := manifest.Config.Digest.String() + layer, err := m.blobFetcher.FetchLayer(repository.Name, digest) + if err != nil { + return err + } + image := &v1.Image{} + if err := json.Unmarshal(layer, image); err != nil { + return err + } + artifact.ExtraAttrs = map[string]interface{}{ + "created": image.Created, + "author": image.Author, + "architecture": image.Architecture, + "os": image.OS, + } + return nil +} diff --git a/src/api/artifact/abstractor/resolver/image/manifest_v2_test.go b/src/api/artifact/abstractor/resolver/image/manifest_v2_test.go new file mode 100644 index 000000000..898926b9f --- /dev/null +++ b/src/api/artifact/abstractor/resolver/image/manifest_v2_test.go @@ -0,0 +1,150 @@ +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package image + +import ( + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/pkg/artifact" + htesting "github.com/goharbor/harbor/src/testing" + "github.com/goharbor/harbor/src/testing/api/artifact/abstractor/blob" + "github.com/stretchr/testify/suite" + "testing" +) + +type manifestV2ResolverTestSuite struct { + suite.Suite + resolver *manifestV2Resolver + repoMgr *htesting.FakeRepositoryManager + blobFetcher *blob.FakeFetcher +} + +func (m *manifestV2ResolverTestSuite) SetupTest() { + m.repoMgr = &htesting.FakeRepositoryManager{} + m.blobFetcher = &blob.FakeFetcher{} + m.resolver = &manifestV2Resolver{ + repoMgr: m.repoMgr, + blobFetcher: m.blobFetcher, + } + +} + +func (m *manifestV2ResolverTestSuite) TestArtifactType() { + m.Assert().Equal(ArtifactTypeImage, m.resolver.ArtifactType()) +} + +func (m *manifestV2ResolverTestSuite) TestResolve() { + content := `{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 1510, + "digest": "sha256:fce289e99eb9bca977dae136fbe2a82b6b7d4c372474c9235adc1741675f587e" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 977, + "digest": "sha256:1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced" + } + ] +}` + config := `{ + "architecture": "amd64", + "config": { + "Hostname": "", + "Domainname": "", + "User": "", + "AttachStdin": false, + "AttachStdout": false, + "AttachStderr": false, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ], + "Cmd": [ + "/hello" + ], + "ArgsEscaped": true, + "Image": "sha256:a6d1aaad8ca65655449a26146699fe9d61240071f6992975be7e720f1cd42440", + "Volumes": null, + "WorkingDir": "", + "Entrypoint": null, + "OnBuild": null, + "Labels": null + }, + "container": "8e2caa5a514bb6d8b4f2a2553e9067498d261a0fd83a96aeaaf303943dff6ff9", + "container_config": { + "Hostname": "8e2caa5a514b", + "Domainname": "", + "User": "", + "AttachStdin": false, + "AttachStdout": false, + "AttachStderr": false, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ], + "Cmd": [ + "/bin/sh", + "-c", + "#(nop) ", + "CMD [\"/hello\"]" + ], + "ArgsEscaped": true, + "Image": "sha256:a6d1aaad8ca65655449a26146699fe9d61240071f6992975be7e720f1cd42440", + "Volumes": null, + "WorkingDir": "", + "Entrypoint": null, + "OnBuild": null, + "Labels": { + + } + }, + "created": "2019-01-01T01:29:27.650294696Z", + "docker_version": "18.06.1-ce", + "history": [ + { + "created": "2019-01-01T01:29:27.416803627Z", + "created_by": "/bin/sh -c #(nop) COPY file:f77490f70ce51da25bd21bfc30cb5e1a24b2b65eb37d4af0c327ddc24f0986a6 in / " + }, + { + "created": "2019-01-01T01:29:27.650294696Z", + "created_by": "/bin/sh -c #(nop) CMD [\"/hello\"]", + "empty_layer": true + } + ], + "os": "linux", + "rootfs": { + "type": "layers", + "diff_ids": [ + "sha256:af0b15c8625bb1938f1d7b17081031f649fd14e6b233688eea3c5483994a66a3" + ] + } +}` + artifact := &artifact.Artifact{} + m.repoMgr.On("Get").Return(&models.RepoRecord{}, nil) + m.blobFetcher.On("FetchLayer").Return([]byte(config), nil) + err := m.resolver.Resolve(nil, []byte(content), artifact) + m.Require().Nil(err) + m.repoMgr.AssertExpectations(m.T()) + m.blobFetcher.AssertExpectations(m.T()) + m.Assert().Equal("amd64", artifact.ExtraAttrs["architecture"].(string)) + m.Assert().Equal("linux", artifact.ExtraAttrs["os"].(string)) +} + +func TestManifestV2ResolverTestSuite(t *testing.T) { + suite.Run(t, &manifestV2ResolverTestSuite{}) +} diff --git a/src/api/artifact/abstractor/resolver/resolver.go b/src/api/artifact/abstractor/resolver/resolver.go new file mode 100644 index 000000000..d5a504e17 --- /dev/null +++ b/src/api/artifact/abstractor/resolver/resolver.go @@ -0,0 +1,58 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resolver + +import ( + "context" + "fmt" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/artifact" +) + +var ( + registry = map[string]Resolver{} +) + +// Resolver resolves the detail information for a specific kind of artifact +type Resolver interface { + // ArtifactType returns the type of artifact that the resolver handles + ArtifactType() string + // Resolve receives the manifest content, resolves the detail information + // from the manifest or the layers referenced by the manifest, and populates + // the detail information into the artifact + Resolve(ctx context.Context, manifest []byte, artifact *artifact.Artifact) error +} + +// Register resolver, one resolver can handle multiple media types for one kind of artifact +func Register(resolver Resolver, mediaTypes ...string) error { + for _, mediaType := range mediaTypes { + _, exist := registry[mediaType] + if exist { + return fmt.Errorf("resolver to handle media type %s already exists", mediaType) + } + registry[mediaType] = resolver + log.Infof("resolver to handle media type %s registered", mediaType) + } + return nil +} + +// Get the resolver according to the media type +func Get(mediaType string) (Resolver, error) { + resolver, exist := registry[mediaType] + if !exist { + return nil, fmt.Errorf("resolver resolves %s not found", mediaType) + } + return resolver, nil +} diff --git a/src/api/artifact/abstractor/resolver/resolver_test.go b/src/api/artifact/abstractor/resolver/resolver_test.go new file mode 100644 index 000000000..457064551 --- /dev/null +++ b/src/api/artifact/abstractor/resolver/resolver_test.go @@ -0,0 +1,54 @@ +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resolver + +import ( + "github.com/stretchr/testify/suite" + "testing" +) + +type resolverTestSuite struct { + suite.Suite +} + +func (r *resolverTestSuite) SetupTest() { + registry = map[string]Resolver{} +} + +func (r *resolverTestSuite) TestRegister() { + // registry a resolver + mediaType := "fake_media_type" + err := Register(nil, mediaType) + r.Assert().Nil(err) + + // try to register a resolver for the existing media type + err = Register(nil, mediaType) + r.Assert().NotNil(err) +} + +func (r *resolverTestSuite) TestGet() { + // registry a resolver + mediaType := "fake_media_type" + err := Register(nil, mediaType) + r.Assert().Nil(err) + + // get the resolver + _, err = Get(mediaType) + r.Assert().Nil(err) + + // get the not exist resolver + _, err = Get("not_existing_media_type") + r.Assert().NotNil(err) +} + +func TestResolverTestSuite(t *testing.T) { + suite.Run(t, &resolverTestSuite{}) +} diff --git a/src/api/artifact/controller.go b/src/api/artifact/controller.go index 4b0f16f85..6c4082c5d 100644 --- a/src/api/artifact/controller.go +++ b/src/api/artifact/controller.go @@ -17,6 +17,9 @@ package artifact import ( "context" "fmt" + "github.com/goharbor/harbor/src/api/artifact/abstractor" + // registry image resolvers + _ "github.com/goharbor/harbor/src/api/artifact/abstractor/resolver/image" "github.com/goharbor/harbor/src/common/utils/log" ierror "github.com/goharbor/harbor/src/internal/error" "github.com/goharbor/harbor/src/pkg/artifact" @@ -29,7 +32,7 @@ import ( var ( // Ctl is a global artifact controller instance - Ctl = NewController(repository.Mgr, artifact.Mgr, tag.Mgr) + Ctl = NewController() ) // Controller defines the operations related with artifacts and tags @@ -63,11 +66,12 @@ type Controller interface { } // NewController creates an instance of the default artifact controller -func NewController(repoMgr repository.Manager, artMgr artifact.Manager, tagMgr tag.Manager) Controller { +func NewController() Controller { return &controller{ - repoMgr: repoMgr, - artMgr: artMgr, - tagMgr: tagMgr, + repoMgr: repository.Mgr, + artMgr: artifact.Mgr, + tagMgr: tag.Mgr, + abstractor: abstractor.NewAbstractor(), } } @@ -76,9 +80,10 @@ func NewController(repoMgr repository.Manager, artMgr artifact.Manager, tagMgr t // for artifacts and tags？？ type controller struct { - repoMgr repository.Manager - artMgr artifact.Manager - tagMgr tag.Manager + repoMgr repository.Manager + artMgr artifact.Manager + tagMgr tag.Manager + abstractor abstractor.Abstractor } func (c *controller) Ensure(ctx context.Context, repositoryID int64, digest string, tags ...string) (bool, int64, error) { @@ -123,7 +128,10 @@ func (c *controller) ensureArtifact(ctx context.Context, repositoryID int64, dig PushTime: time.Now(), } // abstract the specific information for the artifact - c.abstract(ctx, artifact) + if err = c.abstractor.Abstract(ctx, artifact); err != nil { + return false, 0, err + } + // create it id, err := c.artMgr.Create(ctx, artifact) if err != nil { @@ -301,8 +309,3 @@ func (c *controller) assembleTag(ctx context.Context, tag *tm.Tag, option *TagOp // TODO populate label, signature, immutable status for tag return t } - -func (c *controller) abstract(ctx context.Context, artifact *artifact.Artifact) { - // TODO abstract the specific info for the artifact - // handler references -} diff --git a/src/api/artifact/controller_test.go b/src/api/artifact/controller_test.go index b292d4a79..9d70cb626 100644 --- a/src/api/artifact/controller_test.go +++ b/src/api/artifact/controller_test.go @@ -15,32 +15,46 @@ package artifact import ( + "context" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/pkg/artifact" "github.com/goharbor/harbor/src/pkg/q" "github.com/goharbor/harbor/src/pkg/tag/model/tag" htesting "github.com/goharbor/harbor/src/testing" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "testing" "time" ) +type fakeAbstractor struct { + mock.Mock +} + +func (f *fakeAbstractor) Abstract(ctx context.Context, artifact *artifact.Artifact) error { + args := f.Called() + return args.Error(0) +} + type controllerTestSuite struct { suite.Suite - ctl *controller - repoMgr *htesting.FakeRepositoryManager - artMgr *htesting.FakeArtifactManager - tagMgr *htesting.FakeTagManager + ctl *controller + repoMgr *htesting.FakeRepositoryManager + artMgr *htesting.FakeArtifactManager + tagMgr *htesting.FakeTagManager + abstractor *fakeAbstractor } func (c *controllerTestSuite) SetupTest() { c.repoMgr = &htesting.FakeRepositoryManager{} c.artMgr = &htesting.FakeArtifactManager{} c.tagMgr = &htesting.FakeTagManager{} + c.abstractor = &fakeAbstractor{} c.ctl = &controller{ - repoMgr: c.repoMgr, - artMgr: c.artMgr, - tagMgr: c.tagMgr, + repoMgr: c.repoMgr, + artMgr: c.artMgr, + tagMgr: c.tagMgr, + abstractor: c.abstractor, } } @@ -94,10 +108,6 @@ func (c *controllerTestSuite) TestAssembleArtifact() { // TODO check other fields of option } -func (c *controllerTestSuite) TestAbstract() { - // TODO add test case -} - func (c *controllerTestSuite) TestEnsureArtifact() { digest := "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180" @@ -117,16 +127,18 @@ func (c *controllerTestSuite) TestEnsureArtifact() { // reset the mock c.SetupTest() + // the artifact doesn't exist c.repoMgr.On("Get").Return(&models.RepoRecord{ ProjectID: 1, }, nil) - // the artifact doesn't exist c.artMgr.On("List").Return(1, []*artifact.Artifact{}, nil) c.artMgr.On("Create").Return(1, nil) + c.abstractor.On("Abstract").Return(nil) created, id, err = c.ctl.ensureArtifact(nil, 1, digest) c.Require().Nil(err) c.repoMgr.AssertExpectations(c.T()) c.artMgr.AssertExpectations(c.T()) + c.abstractor.AssertExpectations(c.T()) c.True(created) c.Equal(int64(1), id) } @@ -184,11 +196,13 @@ func (c *controllerTestSuite) TestEnsure() { c.artMgr.On("Create").Return(1, nil) c.tagMgr.On("List").Return(1, []*tag.Tag{}, nil) c.tagMgr.On("Create").Return(1, nil) + c.abstractor.On("Abstract").Return(nil) _, id, err := c.ctl.Ensure(nil, 1, digest, "latest") c.Require().Nil(err) c.repoMgr.AssertExpectations(c.T()) c.artMgr.AssertExpectations(c.T()) c.tagMgr.AssertExpectations(c.T()) + c.abstractor.AssertExpectations(c.T()) c.Equal(int64(1), id) } diff --git a/src/testing/api/artifact/abstractor/blob/fetcher.go b/src/testing/api/artifact/abstractor/blob/fetcher.go new file mode 100644 index 000000000..0d533eef1 --- /dev/null +++ b/src/testing/api/artifact/abstractor/blob/fetcher.go @@ -0,0 +1,36 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package blob + +import ( + "github.com/stretchr/testify/mock" +) + +// FakeFetcher is a fake blob fetcher that implement the src/api/artifact/blob.Fetcher interface +type FakeFetcher struct { + mock.Mock +} + +// FetchManifest ... +func (f *FakeFetcher) FetchManifest(repoFullName, digest string) (string, []byte, error) { + args := f.Called(mock.Anything) + return args.String(0), args.Get(1).([]byte), args.Error(2) +} + +// FetchLayer ... +func (f *FakeFetcher) FetchLayer(repoFullName, digest string) (content []byte, err error) { + args := f.Called(mock.Anything) + return args.Get(0).([]byte), args.Error(1) +}