From 7e4b26b220e762a6e25cdbdcf39d2513975a894e Mon Sep 17 00:00:00 2001 From: Roooocky <48276426+ln23415@users.noreply.github.com> Date: Fri, 8 Jul 2022 23:08:28 +0800 Subject: [PATCH] Add new feature for supporting WebAssembly artifact (#16931) support wasm Signed-off-by: ln23415 --- icons/wasm.png | Bin 0 -> 3275 bytes src/controller/artifact/abstractor.go | 6 + src/controller/artifact/controller.go | 2 + .../artifact/processor/wasm/wasm.go | 135 +++++++++++++ .../artifact/processor/wasm/wasm_test.go | 180 ++++++++++++++++++ src/controller/icon/controller.go | 4 + src/lib/icon/const.go | 1 + 7 files changed, 328 insertions(+) create mode 100644 icons/wasm.png create mode 100644 src/controller/artifact/processor/wasm/wasm.go create mode 100644 src/controller/artifact/processor/wasm/wasm_test.go diff --git a/icons/wasm.png b/icons/wasm.png new file mode 100644 index 0000000000000000000000000000000000000000..9df4126544a9fcf7af6216535c3a4235c7ef0373 GIT binary patch literal 3275 zcmc&%=Q|s2)Q;AU6?;VOQ88=NphQEBP+Drns-ITGYE`Kkt)i->c8yZ8Yp=$hK}n5L zM2gt8YBV-)-p}tJ@P2rn`@^};b)Iuy=iJXZu@>fVW=4KS006*jYy`VYLHGYX^t6=Q zmy}`+05ByP!){qW%i78g@dm>MaqY^dP1?CzWm`SN->VljSf#w2@MTXvHCRfg@Y`Jg ztnXRKaM~5ie6TRY+=+Qp%w}X{A#P1gVzs!uazOD!|C-67xt5fQ31 zd!OkP-If@7w$(n=!WEn3n%*w9Mr;G{S#cE0rf#ikC@7?EX&V~f3KVR5_S;J~^&BEp zD}l9?-3W@ViF7#WFM*_+wyJy~ z=kQohAnLF_ZBe1Obig)mi<{AdhT-+V{>~Zn{F^=jrRxm^Gk#nJq8crvA@l%iIniHj z=R->fMvtB2ABeM>!c>Rr$*r!R>x83?+x)f|%U*0PADIL_+6YlFf1#c+?~!TlB->NZ z1;kJy{-=3thj>HW>qeWs@XS+ldg1bCqSL8JlU;%i7>w%G3%&*TEP6XfVDm|4`^nJ} zdWOXw5&+#^%L^R0(NVr2jDm{VD0T=*f}1Jd^$(2J4Q$16##$|-)wZLzhDHc631wcY ziy5#$_bLzEE{}0T99A4HH53<&^{_}q3?fCC5By0C8Z~Do^(^>CM9s8EqIBVR$iO&u z8}F%uS_?Y8!W~)20Sg3MD``F}3YBr~zvPGzjswx>^Zx&VIL+8_D>3&hH4>U`>I{UUUn#NRXiFcpsHrfgXCaaCr*Why zu!kjMY3z3nKgpXq1iyfaajHU&i^C*VLk@AB-rd0;gH=zXjyUOZ!G~WhaJ2lHDxsJE z7+sFb6igTPXy|?>J9kff2fy56Yd5JaBlbO3K#1H|Jv@u|n^OC`uWQ}Gv^;WEaNdNK zs5g;6``lYCj@!WW(-}DqUn}XI!}>m3Z7*Q-_&=$)x-exf>@Lgq6|CsLfmwCKhJ^F3Twf$y)J-8?=&Why8H?z80PvxJn;&hl~`E@N+- z*=1yXgBtBKl&ZeHe&+D>QKpIgNx!f`=$Jq|@8Sq^Z~1a5hG|8_jl+Z+f5L-}&b?eJTI+)wjw< zwtq!1y6AP0XSg8ivJUH113Y(JhHtUB1Y=vH?@(JbBkOWjHKzCdDrrB=Kh{MHZ&@EB zYG*t$E>iIi*N${`HFZg&0X(E%+hzmidxe9r#4hE_is`j~iw5TvWf_R>nQsxq>ImDE z7EG>=PdD4ph?Q}|*N~n_rkrZ@a(x$Qz9+=>R{k<)^&joGVum~vdf#>@313lO=o<6`=T$+U^<~-9dsIcNfS8W=3~gH)++F z95#8tWWATS7RkmK@4Pz3KpMQwy!jC1*^zIzE0GOFEp};EzN;5d9TG=h>SnL}XQ!}z zjnWum4wCq7c}HW+{(Io6;F@}Ue)4>uYV;_`PwX)fEvi%E%bJ(m(V!7c*P~wLk>*$S znxIzYNWw`;@nUJ(wC&8RLsPU(@I!UK(4fYHZ5Dw-B%238&)|zK3`Z4xlBMazO%z_! zkbG({(C~*oO>P#I;@+zW7Fm15gG*HXSB}BQTx_-cJ;5K zq)jh-t;?zF8EiQl0hJ+SMNKf3)?6BE(Mt0|sC29_c7gUb=-vb;mr|plgrugOT`Sbx zeS`|5XVcuoo~hPE;h&fsAA^uris6@Y12^e>>f6 z%+yz>FI)r@#0`Fpqf&j(qm+=xjKQjbVZu)XxRfFfFXXnXDczr1@z**0qh*bwRHAg} z`l4%Ny;pNYk2rk)z8|DjW3$3BU9vO7yz*V?qd3EQ&e-e0@{fvl$VOgjCE`_P^O6w+ z6S{nrw?6%j((@|~?iqmVSvPkFJf zSELfPjmeoNXGO>M2Rvi|fcKg&(jIL6cUQlZ>fPgB=(r*SS&*LL%^-knZW-$`TI{n$ z3oUYl>|f6i&NNrNA^fy-)IaW2{QFnv7C9^K;Vf$R-b4T-&-})FK=|QB#_b2+{?nJ{ zxe6c)Dpv1wbWOqmNZeIg^ZEz$uLE=o3$%H@aOt)8EZ=;tmo>h&Ugt7f{&7frf0hre z@L=@w!o>^H;3w#K&N~#9eGu1L}0F z$e)#yuf95}jN=uv=cgnGm^dRR&yPJ{n#j;kgx5m9AzZZI+ZQ)^Zts{{-Nf$NSJ3o` zVcfgsJh&+8T;HOmn{3V29XQVOZEV74Zm{OzXU--+=pog`sDW29P;KaeS3>F}>CFg~ zb+ys6Y-#SZD|jq%Ae8a}4GHmr7~>s1Gzn`^l*uCQdG5IlWiA_f%rlVI&Iclo!E+RM zan4d8E+!cmC5XDTa@NKZ|I#6#0gFDUl6l29o@G_Ce_EanazFUeN2{~isz!P5}bah&)?K6Wq)2xLf7ZVM1V&YZ~S(k4xc(inhqDrQ-AJEK$BLs;G(~j(F=f4Bj zJSif8{;x_7Il+It_9(Dtw?|%AB^x|fJsOT@popgx*+^ixnWpoZ}G5S&&6Y;u$e_8xrV6J!{A477>1}L+ zM7|QE7KAVHyxd~MEj7Kwq!)%h;04f%b6DWRXE1xr%z`-Ch1)USypq00s9@yWr8PY^ z2YMP}qhmhA`^JU6+^ML$x}_*LeZDEQ6SSS1i4tSaD@KpqR8xtX?+S(=fyLawl!yZEeJy7#`(0mYsN9D+P4hn=+t)`O33{dn zk;^})tTd`Q3`O4!QcM~qJD%F9V8koy0!2`qq}dy1&!5OsE3jC^>#_CHQ(OLDI1+0A XGVqsfnh-`=QUJz==CBHVWaR$<6&Nxd literal 0 HcmV?d00001 diff --git a/src/controller/artifact/abstractor.go b/src/controller/artifact/abstractor.go index f8c52f842..84120c688 100644 --- a/src/controller/artifact/abstractor.go +++ b/src/controller/artifact/abstractor.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/goharbor/harbor/src/controller/artifact/processor/wasm" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg" @@ -126,6 +127,11 @@ func (a *abstractor) abstractManifestV2Metadata(artifact *artifact.Artifact, con } // use the "manifest.config.mediatype" as the media type of the artifact artifact.MediaType = manifest.Config.MediaType + + if manifest.Annotations[wasm.AnnotationVariantKey] == wasm.AnnotationVariantValue || manifest.Annotations[wasm.AnnotationHandlerKey] == wasm.AnnotationHandlerValue { + artifact.MediaType = wasm.MediaType + } + // set size artifact.Size = int64(len(content)) + manifest.Config.Size for _, layer := range manifest.Layers { diff --git a/src/controller/artifact/controller.go b/src/controller/artifact/controller.go index 7b2ecfe99..bf0d76f56 100644 --- a/src/controller/artifact/controller.go +++ b/src/controller/artifact/controller.go @@ -28,6 +28,7 @@ import ( "github.com/goharbor/harbor/src/controller/artifact/processor/chart" "github.com/goharbor/harbor/src/controller/artifact/processor/cnab" "github.com/goharbor/harbor/src/controller/artifact/processor/image" + "github.com/goharbor/harbor/src/controller/artifact/processor/wasm" "github.com/goharbor/harbor/src/lib/icon" "github.com/goharbor/harbor/src/controller/artifact/processor" @@ -72,6 +73,7 @@ var ( image.ArtifactTypeImage: icon.DigestOfIconImage, chart.ArtifactTypeChart: icon.DigestOfIconChart, cnab.ArtifactTypeCNAB: icon.DigestOfIconCNAB, + wasm.ArtifactTypeWASM: icon.DigestOfIconWASM, } ) diff --git a/src/controller/artifact/processor/wasm/wasm.go b/src/controller/artifact/processor/wasm/wasm.go new file mode 100644 index 000000000..ad0fbc4a2 --- /dev/null +++ b/src/controller/artifact/processor/wasm/wasm.go @@ -0,0 +1,135 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wasm + +import ( + "context" + "encoding/json" + "github.com/goharbor/harbor/src/controller/artifact/processor" + "github.com/goharbor/harbor/src/controller/artifact/processor/base" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/opencontainers/image-spec/specs-go/v1" +) + +// const definitions +const ( + // ArtifactTypeWASM is the artifact type for image + ArtifactTypeWASM = "WASM" + AdditionTypeBuildHistory = "BUILD_HISTORY" + + // AnnotationVariantKey and AnnotationVariantValue is available key-value pair to identify an annotation fashion wasm artifact + AnnotationVariantKey = "module.wasm.image/variant" + AnnotationVariantValue = "compat" + + // AnnotationHandlerKey and AnnotationHandlerValue is another available key-value pair to identify an annotation fashion wasm artifact + AnnotationHandlerKey = "run.oci.handler" + AnnotationHandlerValue = "wasm" + + MediaType = "application/vnd.wasm.config.v1+json" +) + +func init() { + pc := &Processor{} + pc.ManifestProcessor = base.NewManifestProcessor() + mediaTypes := []string{ + MediaType, + } + if err := processor.Register(pc, mediaTypes...); err != nil { + log.Errorf("failed to register processor for media type %v: %v", mediaTypes, err) + return + } +} + +// Processor processes image with OCI manifest and docker v2 manifest +type Processor struct { + *base.ManifestProcessor +} + +func (m *Processor) AbstractMetadata(ctx context.Context, art *artifact.Artifact, manifestBody []byte) error { + art.ExtraAttrs = map[string]interface{}{} + manifest := &v1.Manifest{} + if err := json.Unmarshal(manifestBody, manifest); err != nil { + return err + } + + if art.ExtraAttrs == nil { + art.ExtraAttrs = map[string]interface{}{} + } + if manifest.Annotations[AnnotationVariantKey] == AnnotationVariantValue || manifest.Annotations[AnnotationHandlerKey] == AnnotationHandlerValue { + // for annotation way + config := &v1.Image{} + if err := m.UnmarshalConfig(ctx, art.RepositoryName, manifestBody, config); err != nil { + return err + } + art.ExtraAttrs["manifest.config.mediaType"] = manifest.Config.MediaType + art.ExtraAttrs["created"] = config.Created + art.ExtraAttrs["architecture"] = config.Architecture + art.ExtraAttrs["os"] = config.OS + art.ExtraAttrs["config"] = config.Config + // if the author is null, try to get it from labels: + // https://docs.docker.com/engine/reference/builder/#maintainer-deprecated + author := config.Author + if len(author) == 0 && len(config.Config.Labels) > 0 { + author = config.Config.Labels["maintainer"] + } + art.ExtraAttrs["author"] = author + } else { + // for wasm-to-oci way + art.ExtraAttrs["manifest.config.mediaType"] = MediaType + if len(manifest.Layers) > 0 { + art.ExtraAttrs["manifest.layers.mediaType"] = manifest.Layers[0].MediaType + art.ExtraAttrs["org.opencontainers.image.title"] = manifest.Layers[0].Annotations["org.opencontainers.image.title"] + } + } + return nil +} + +func (m *Processor) AbstractAddition(ctx context.Context, artifact *artifact.Artifact, addition string) (*processor.Addition, error) { + if addition != AdditionTypeBuildHistory { + return nil, errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage("addition %s isn't supported for %s(manifest version 2)", addition, ArtifactTypeWASM) + } + + mani, _, err := m.RegCli.PullManifest(artifact.RepositoryName, artifact.Digest) + if err != nil { + return nil, err + } + _, content, err := mani.Payload() + if err != nil { + return nil, err + } + config := &v1.Image{} + if err = m.ManifestProcessor.UnmarshalConfig(ctx, artifact.RepositoryName, content, config); err != nil { + return nil, err + } + content, err = json.Marshal(config.History) + if err != nil { + return nil, err + } + return &processor.Addition{ + Content: content, + ContentType: "application/json; charset=utf-8", + }, nil +} + +func (m *Processor) GetArtifactType(ctx context.Context, artifact *artifact.Artifact) string { + return ArtifactTypeWASM +} + +func (m *Processor) ListAdditionTypes(ctx context.Context, artifact *artifact.Artifact) []string { + return []string{AdditionTypeBuildHistory} +} diff --git a/src/controller/artifact/processor/wasm/wasm_test.go b/src/controller/artifact/processor/wasm/wasm_test.go new file mode 100644 index 000000000..acb57d92c --- /dev/null +++ b/src/controller/artifact/processor/wasm/wasm_test.go @@ -0,0 +1,180 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wasm + +import ( + "bytes" + "encoding/json" + + "io/ioutil" + "strings" + "testing" + + "github.com/docker/distribution/manifest/schema2" + + "github.com/goharbor/harbor/src/controller/artifact/processor/base" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/goharbor/harbor/src/testing/mock" + "github.com/goharbor/harbor/src/testing/pkg/registry" + "github.com/stretchr/testify/suite" +) + +var ( + // For OCI fashion wasm artifact + oci_manifest = `{ + "schemaVersion":2, + "config":{ + "mediaType":"application/vnd.wasm.config.v1+json", + "digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "size":2 + }, + "layers":[ + { + "mediaType":"application/vnd.wasm.content.layer.v1+wasm", + "digest":"sha256:d43012458290e4e2a350055bbe4a9f49fd4fb6b51d412089301e63ea4397ab4f", + "size":3951005, + "annotations":{ + "org.opencontainers.image.title":"test.wasm" + } + } + ] +}` + oci_config = `{}` + + // For annotation fashion wasm artifact + annnotated_manifest = `{ + "schemaVersion":2, + "mediaType":"application/vnd.oci.image.manifest.v1+json", + "config":{ + "mediaType":"application/vnd.oci.image.config.v1+json", + "digest":"sha256:6fd90b7cd05366c82ca32a3ff259e62dcb15b3b5e9672fe7d45609f29b6c1e95", + "size":637 + }, + "layers":[ + { + "mediaType":"application/vnd.oci.image.layer.v1.tar+gzip", + "digest":"sha256:818ab7fdcd8d16270f01795d6aee7af0d1a06c71ce2cd3c1e8d8f946e9475450", + "size":500361 + } + ], + "annotations":{ + "module.wasm.image/variant":"compat", + "org.opencontainers.image.base.digest":"", + "org.opencontainers.image.base.name":"" + } +}` + annnotated_config = `{ + "created":"2022-03-02T09:02:41.01773982Z", + "architecture":"amd64", + "os":"linux", + "config":{ + "Env":[ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ], + "Cmd":[ + "/sleep.wasm" + ], + "Labels":{ + "io.buildah.version":"1.25.0-dev" + } + }, + "rootfs":{ + "type":"layers", + "diff_ids":[ + "sha256:65885aa5fd4c157de98241de75669c4bf0d6f17d220c40069de31a572371a80d" + ] + }, + "history":[ + { + "created":"2022-03-02T09:02:41.011039932Z", + "created_by":"/bin/sh -c #(nop) COPY file:9fc00231cd29a2b8f76cfeaa9bc2355a7df54585b51d1f5dc98daf51557614b9 in / ", + "empty_layer":true + }, + { + "created":"2022-03-02T09:02:41.043350231Z", + "created_by":"/bin/sh -c #(nop) CMD [\"/sleep.wasm\"]" + } + ] +}` +) + +type WASMProcessorTestSuite struct { + suite.Suite + processor *Processor + regCli *registry.FakeClient +} + +func (m *WASMProcessorTestSuite) SetupTest() { + m.regCli = ®istry.FakeClient{} + m.processor = &Processor{} + m.processor.ManifestProcessor = &base.ManifestProcessor{RegCli: m.regCli} +} + +func (m *WASMProcessorTestSuite) TestAbstractMetadataForAnnotationFashion() { + artifact := &artifact.Artifact{} + m.regCli.On("PullBlob", mock.Anything, mock.Anything).Return(0, ioutil.NopCloser(bytes.NewReader([]byte(annnotated_config))), nil) + err := m.processor.AbstractMetadata(nil, artifact, []byte(annnotated_manifest)) + m.Require().Nil(err) + m.NotNil(artifact.ExtraAttrs["created"]) + m.Equal("amd64", artifact.ExtraAttrs["architecture"]) + m.Equal("linux", artifact.ExtraAttrs["os"]) + m.NotNil(artifact.ExtraAttrs["config"]) + m.regCli.AssertExpectations(m.T()) + +} + +func (m *WASMProcessorTestSuite) TestAbstractMetadataForOCIFashion() { + artifact := &artifact.Artifact{} + err := m.processor.AbstractMetadata(nil, artifact, []byte(oci_manifest)) + m.Require().Nil(err) + m.NotNil(artifact.ExtraAttrs["org.opencontainers.image.title"]) + m.Equal(MediaType, artifact.ExtraAttrs["manifest.config.mediaType"]) + m.NotNil(artifact.ExtraAttrs["manifest.layers.mediaType"]) + m.regCli.AssertExpectations(m.T()) +} + +func (m *WASMProcessorTestSuite) TestAbstractAdditionForAnnotationFashion() { + // unknown addition + _, err := m.processor.AbstractAddition(nil, nil, "unknown_addition") + m.True(errors.IsErr(err, errors.BadRequestCode)) + + // build history + artifact := &artifact.Artifact{} + + manifest := schema2.Manifest{} + err = json.Unmarshal([]byte(annnotated_manifest), &manifest) + deserializedManifest, err := schema2.FromStruct(manifest) + m.Require().Nil(err) + m.regCli.On("PullManifest").Return(deserializedManifest, "", nil) + m.regCli.On("PullBlob").Return(0, ioutil.NopCloser(strings.NewReader(annnotated_config)), nil) + addition, err := m.processor.AbstractAddition(nil, artifact, AdditionTypeBuildHistory) + m.Require().Nil(err) + m.Equal("application/json; charset=utf-8", addition.ContentType) + m.Equal(`[{"created":"2022-03-02T09:02:41.011039932Z","created_by":"/bin/sh -c #(nop) COPY file:9fc00231cd29a2b8f76cfeaa9bc2355a7df54585b51d1f5dc98daf51557614b9 in / ","empty_layer":true},{"created":"2022-03-02T09:02:41.043350231Z","created_by":"/bin/sh -c #(nop) CMD [\"/sleep.wasm\"]"}]`, string(addition.Content)) +} + +func (m *WASMProcessorTestSuite) TestGetArtifactType() { + m.Assert().Equal(ArtifactTypeWASM, m.processor.GetArtifactType(nil, nil)) +} + +func (m *WASMProcessorTestSuite) TestListAdditionTypes() { + additions := m.processor.ListAdditionTypes(nil, nil) + m.EqualValues([]string{AdditionTypeBuildHistory}, additions) +} + +func TestManifestV2ProcessorTestSuite(t *testing.T) { + suite.Run(t, &WASMProcessorTestSuite{}) +} diff --git a/src/controller/icon/controller.go b/src/controller/icon/controller.go index 34ac9eea8..afa19a56f 100644 --- a/src/controller/icon/controller.go +++ b/src/controller/icon/controller.go @@ -60,6 +60,10 @@ var ( path: "./icons/nydus.png", resize: false, }, + icon.DigestOfIconWASM: { + path: "./icons/wasm.png", + resize: true, + }, icon.DigestOfIconDefault: { path: "./icons/default.png", resize: true, diff --git a/src/lib/icon/const.go b/src/lib/icon/const.go index 0c3f1c525..69c7b691c 100644 --- a/src/lib/icon/const.go +++ b/src/lib/icon/const.go @@ -6,6 +6,7 @@ const ( DigestOfIconChart = "sha256:61cf3a178ff0f75bf08a25d96b75cf7355dc197749a9f128ed3ef34b0df05518" DigestOfIconCNAB = "sha256:089bdda265c14d8686111402c8ad629e8177a1ceb7dcd0f7f39b6480f623b3bd" DigestOfIconDefault = "sha256:da834479c923584f4cbcdecc0dac61f32bef1d51e8aae598cf16bd154efab49f" + DigestOfIconWASM = "sha256:badd7693bcaf115be202748241dd0ea6ee3b0524bfab9ac22d1e1c43721afec6" // ToDo add the accessories images DigestOfIconAccDefault = ""