From ee6f61c50274734de4cfb970d88a94fbc9a8b28f Mon Sep 17 00:00:00 2001 From: zycupup <70937795+zycupup@users.noreply.github.com> Date: Fri, 19 Jan 2024 14:15:49 +0800 Subject: [PATCH] feat: volc cr adapter (#19456) feat: support volcEngine replication Signed-off-by: zhuyuchen.1 --- src/go.mod | 8 +- src/go.sum | 9 + .../job/impl/replication/replication.go | 2 + src/pkg/reg/adapter/volcenginecr/adapter.go | 167 +++++++++ .../reg/adapter/volcenginecr/adapter_test.go | 187 ++++++++++ .../adapter/volcenginecr/artifact_registry.go | 205 +++++++++++ .../volcenginecr/artifact_registry_test.go | 55 +++ src/pkg/reg/adapter/volcenginecr/auth.go | 93 +++++ src/pkg/reg/adapter/volcenginecr/auth_test.go | 29 ++ src/pkg/reg/adapter/volcenginecr/consts.go | 33 ++ src/pkg/reg/adapter/volcenginecr/helper.go | 66 ++++ .../reg/adapter/volcenginecr/helper_test.go | 56 +++ src/pkg/reg/adapter/volcenginecr/volccr.go | 334 ++++++++++++++++++ .../reg/adapter/volcenginecr/volccr_test.go | 70 ++++ src/pkg/reg/manager.go | 6 +- src/pkg/reg/model/registry.go | 1 + .../app/shared/services/endpoint.service.ts | 1 + 17 files changed, 1317 insertions(+), 5 deletions(-) create mode 100644 src/pkg/reg/adapter/volcenginecr/adapter.go create mode 100644 src/pkg/reg/adapter/volcenginecr/adapter_test.go create mode 100644 src/pkg/reg/adapter/volcenginecr/artifact_registry.go create mode 100644 src/pkg/reg/adapter/volcenginecr/artifact_registry_test.go create mode 100644 src/pkg/reg/adapter/volcenginecr/auth.go create mode 100644 src/pkg/reg/adapter/volcenginecr/auth_test.go create mode 100644 src/pkg/reg/adapter/volcenginecr/consts.go create mode 100644 src/pkg/reg/adapter/volcenginecr/helper.go create mode 100644 src/pkg/reg/adapter/volcenginecr/helper_test.go create mode 100644 src/pkg/reg/adapter/volcenginecr/volccr.go create mode 100644 src/pkg/reg/adapter/volcenginecr/volccr_test.go diff --git a/src/go.mod b/src/go.mod index dd2f7adb7..86913fd02 100644 --- a/src/go.mod +++ b/src/go.mod @@ -21,12 +21,12 @@ require ( github.com/go-asn1-ber/asn1-ber v1.5.1 github.com/go-ldap/ldap/v3 v3.2.4 github.com/go-openapi/errors v0.20.4 - github.com/go-openapi/loads v0.21.2 // indirect + github.com/go-openapi/loads v0.21.2 github.com/go-openapi/runtime v0.26.2 - github.com/go-openapi/spec v0.20.11 // indirect + github.com/go-openapi/spec v0.20.11 github.com/go-openapi/strfmt v0.21.8 github.com/go-openapi/swag v0.22.7 - github.com/go-openapi/validate v0.22.3 // indirect + github.com/go-openapi/validate v0.22.3 github.com/go-redis/redis/v8 v8.11.4 github.com/gocarina/gocsv v0.0.0-20210516172204-ca9e8a8ddea8 github.com/gocraft/work v0.5.1 @@ -53,6 +53,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/tencentcloud/tencentcloud-sdk-go v1.0.62 github.com/vmihailenco/msgpack/v5 v5.4.1 + github.com/volcengine/volcengine-go-sdk v1.0.97 go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.46.1 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 go.opentelemetry.io/otel v1.21.0 @@ -153,6 +154,7 @@ require ( github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/volcengine/volc-sdk-golang v1.0.23 // indirect go.mongodb.org/mongo-driver v1.13.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect go.opentelemetry.io/otel/metric v1.21.0 // indirect diff --git a/src/go.sum b/src/go.sum index e9c9e8e69..2edddf473 100644 --- a/src/go.sum +++ b/src/go.sum @@ -74,6 +74,7 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= @@ -246,6 +247,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= @@ -278,6 +280,7 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -403,6 +406,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -595,6 +599,10 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/volcengine/volc-sdk-golang v1.0.23 h1:anOslb2Qp6ywnsbyq9jqR0ljuO63kg9PY+4OehIk5R8= +github.com/volcengine/volc-sdk-golang v1.0.23/go.mod h1:AfG/PZRUkHJ9inETvbjNifTDgut25Wbkm2QoYBTbvyU= +github.com/volcengine/volcengine-go-sdk v1.0.97 h1:JykYagPlleFuFIrk90uigS1UyIZPRIYX6TnC6FErWP4= +github.com/volcengine/volcengine-go-sdk v1.0.97/go.mod h1:oht5AKDJsk0fY6tV2ViqaVlOO14KSRmXZlI8ikK60Tg= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= @@ -893,6 +901,7 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/src/jobservice/job/impl/replication/replication.go b/src/jobservice/job/impl/replication/replication.go index 740ea3797..453fa5e03 100644 --- a/src/jobservice/job/impl/replication/replication.go +++ b/src/jobservice/job/impl/replication/replication.go @@ -51,6 +51,8 @@ import ( _ "github.com/goharbor/harbor/src/pkg/reg/adapter/quay" // import tencentcr adapter _ "github.com/goharbor/harbor/src/pkg/reg/adapter/tencentcr" + // register the VolcEngine CR Registry adapter + _ "github.com/goharbor/harbor/src/pkg/reg/adapter/volcenginecr" "github.com/goharbor/harbor/src/pkg/reg/model" ) diff --git a/src/pkg/reg/adapter/volcenginecr/adapter.go b/src/pkg/reg/adapter/volcenginecr/adapter.go new file mode 100644 index 000000000..106ba76cb --- /dev/null +++ b/src/pkg/reg/adapter/volcenginecr/adapter.go @@ -0,0 +1,167 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcenginecr + +import ( + "errors" + "path" + "strings" + + volcCR "github.com/volcengine/volcengine-go-sdk/service/cr" + "github.com/volcengine/volcengine-go-sdk/volcengine" + "github.com/volcengine/volcengine-go-sdk/volcengine/credentials" + volcSession "github.com/volcengine/volcengine-go-sdk/volcengine/session" + + "github.com/goharbor/harbor/src/lib/log" + adp "github.com/goharbor/harbor/src/pkg/reg/adapter" + "github.com/goharbor/harbor/src/pkg/reg/adapter/native" + "github.com/goharbor/harbor/src/pkg/reg/model" + "github.com/goharbor/harbor/src/pkg/reg/util" + "github.com/goharbor/harbor/src/pkg/registry/auth/bearer" +) + +func init() { + if err := adp.RegisterFactory(model.RegistryTypeVolcCR, new(factory)); err != nil { + log.Errorf("failed to register factory for %s: %v", model.RegistryTypeVolcCR, err) + return + } + log.Infof("the factory for adapter %s registered", model.RegistryTypeVolcCR) +} + +type factory struct{} + +/** + * Implement Factory Interface +**/ +var _ adp.Factory = &factory{} + +type adapter struct { + *native.Adapter + registryName *string + volcCrClient *volcCR.CR + registry *model.Registry +} + +// Create ... +func (f *factory) Create(r *model.Registry) (adp.Adapter, error) { + return newAdapter(r) +} + +// AdapterPattern ... +func (f *factory) AdapterPattern() *model.AdapterPattern { + return getAdapterInfo() +} + +func getAdapterInfo() *model.AdapterPattern { + return &model.AdapterPattern{} +} + +func newAdapter(registry *model.Registry) (a *adapter, err error) { + // get region and registryName from url + region, registryName, err := getRegionRegistryName(registry.URL) + if err != nil { + log.Errorf("getRegion failed. error=%v", err) + return nil, err + } + + // Create VolcCR API client + config := volcengine.NewConfig(). + WithCredentials(credentials.NewStaticCredentials(registry.Credential.AccessKey, registry.Credential.AccessSecret, "")). + WithRegion(region) + sess, err := volcSession.NewSession(config) + if err != nil { + log.Errorf("getSession error. error=%v", err) + return nil, err + } + client := volcCR.New(sess) + + // Get AuthorizationToken for docker login + bearRealm, bearService, err := getRealmService(registry.URL, registry.Insecure) + if err != nil { + log.Error("fail to ping the registry", "url", registry.URL) + return nil, err + } + cred := NewAuth(client, registryName) + var transport = util.GetHTTPTransport(registry.Insecure) + authorizer := bearer.NewAuthorizer(bearRealm, bearService, cred, transport) + + return &adapter{ + registry: registry, + registryName: ®istryName, + volcCrClient: client, + Adapter: native.NewAdapterWithAuthorizer(registry, authorizer), + }, nil +} + +func (a *adapter) Info() (info *model.RegistryInfo, err error) { + info = &model.RegistryInfo{ + Type: model.RegistryTypeVolcCR, + SupportedResourceTypes: []string{ + model.ResourceTypeImage, + }, + SupportedResourceFilters: []*model.FilterStyle{ + { + Type: model.FilterTypeName, + Style: model.FilterStyleTypeText, + }, + { + Type: model.FilterTypeTag, + Style: model.FilterStyleTypeText, + }, + }, + SupportedTriggers: []string{ + model.TriggerTypeManual, + model.TriggerTypeScheduled, + }, + } + return +} + +func (a *adapter) PrepareForPush(resources []*model.Resource) (err error) { + for _, resource := range resources { + if resource == nil { + return errors.New("the resource cannot be null") + } + if resource.Metadata == nil { + return errors.New("[volcengine-cr.PrepareForPush] the metadata of resource cannot be null") + } + if resource.Metadata.Repository == nil { + return errors.New("[volcengine-cr.PrepareForPush] the namespace of resource cannot be null") + } + if len(resource.Metadata.Repository.Name) == 0 { + return errors.New("[volcengine-cr.PrepareForPush] the name of the namespace cannot be null") + } + var paths = strings.Split(resource.Metadata.Repository.Name, "/") + if len(paths) < 2 { + return errors.New("[volcengine-cr.PrepareForPush] the name of the repository and namespace cannot be null") + } + var namespace = paths[0] + var repository = path.Join(paths[1:]...) + + log.Debugf("namespace=%s", namespace) + err = a.createNamespace(namespace) + if err != nil { + log.Errorf("PrepareForPush error :%v", err) + return + } + log.Debugf("namespace=%s, repository=%s", namespace, repository) + err = a.createRepository(namespace, repository) + if err != nil { + log.Errorf("PrepareForPush error :%v", err) + return + } + } + return +} diff --git a/src/pkg/reg/adapter/volcenginecr/adapter_test.go b/src/pkg/reg/adapter/volcenginecr/adapter_test.go new file mode 100644 index 000000000..5ddb5b6b7 --- /dev/null +++ b/src/pkg/reg/adapter/volcenginecr/adapter_test.go @@ -0,0 +1,187 @@ +package volcenginecr + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "github.com/goharbor/harbor/src/common/utils/test" + adp "github.com/goharbor/harbor/src/pkg/reg/adapter" + "github.com/goharbor/harbor/src/pkg/reg/adapter/native" + "github.com/goharbor/harbor/src/pkg/reg/model" + "github.com/stretchr/testify/assert" + volcCR "github.com/volcengine/volcengine-go-sdk/service/cr" + "github.com/volcengine/volcengine-go-sdk/volcengine" + "github.com/volcengine/volcengine-go-sdk/volcengine/credentials" + volcSession "github.com/volcengine/volcengine-go-sdk/volcengine/session" +) + +func getMockAdapter_withoutCred(t *testing.T, hasCred, health bool) (*adapter, *httptest.Server) { + server := test.NewServer( + &test.RequestHandlerMapping{ + Method: http.MethodGet, + Pattern: "/v2/", + Handler: func(w http.ResponseWriter, r *http.Request) { + fmt.Println(r.Method, r.URL) + if health { + w.WriteHeader(http.StatusOK) + } else { + w.WriteHeader(http.StatusBadRequest) + } + }, + }, + &test.RequestHandlerMapping{ + Method: http.MethodGet, + Pattern: "/", + Handler: func(w http.ResponseWriter, r *http.Request) { + fmt.Println(r.Method, r.URL) + w.WriteHeader(http.StatusOK) + }, + }, + &test.RequestHandlerMapping{ + Method: http.MethodPost, + Pattern: "/", + Handler: func(w http.ResponseWriter, r *http.Request) { + fmt.Println(r.Method, r.URL) + if buf, e := ioutil.ReadAll(&io.LimitedReader{R: r.Body, N: 80}); e == nil { + fmt.Println("\t", string(buf)) + } + w.WriteHeader(http.StatusOK) + }, + }, + ) + + registry := &model.Registry{ + Type: model.RegistryTypeVolcCR, + URL: server.URL, + } + if hasCred { + registry.Credential = &model.Credential{ + AccessKey: "MockAccessKey", + AccessSecret: "MockAccessSecret", + } + } + name := "test-registry" + config := volcengine.NewConfig(). + WithCredentials(credentials.NewStaticCredentials("", "", "")). + WithRegion("cn-beijing") + sess, _ := volcSession.NewSession(config) + client := volcCR.New(sess) + return &adapter{ + Adapter: native.NewAdapter(registry), + registryName: &name, + volcCrClient: client, + registry: registry, + }, server +} + +func TestAdapter_NewAdapter_InvalidURL(t *testing.T) { + factory, err := adp.GetFactory("BadName") + assert.Nil(t, factory) + assert.Error(t, err) + + factory, err = adp.GetFactory(model.RegistryTypeVolcCR) + assert.NoError(t, err) + assert.NotNil(t, factory) + adapter, err := factory.Create(&model.Registry{ + Type: model.RegistryTypeVolcCR, + Credential: &model.Credential{}, + }) + assert.Error(t, err) + assert.Nil(t, adapter) +} + +func TestAdapter_NewAdapter_PingFailed(t *testing.T) { + factory, _ := adp.GetFactory(model.RegistryTypeVolcCR) + adapter, err := factory.Create(&model.Registry{ + Type: model.RegistryTypeVolcCR, + Credential: &model.Credential{}, + URL: "https://cr-test-cn-beijing.cr.volces.com", + }) + assert.Error(t, err) + assert.Nil(t, adapter) +} + +func TestAdapter_Info(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + info, err := a.Info() + assert.Nil(t, err) + assert.NotNil(t, info) + + assert.EqualValues(t, 1, len(info.SupportedResourceTypes)) + assert.EqualValues(t, model.ResourceTypeImage, info.SupportedResourceTypes[0]) +} + +func TestAdapter_PrepareForPush(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + resources := []*model.Resource{ + { + Type: model.ResourceTypeImage, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: "busybox", + }, + }, + }, + } + + err := a.PrepareForPush(resources) + assert.Error(t, err) +} + +func TestAdapter_PrepareForPush_NilResource(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + var resources = []*model.Resource{nil} + + err := a.PrepareForPush(resources) + assert.Error(t, err) +} + +func TestAdapter_PrepareForPush_NilMeta(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + resources := []*model.Resource{ + { + Type: model.ResourceTypeImage, + }, + } + + err := a.PrepareForPush(resources) + assert.Error(t, err) +} + +func TestAdapter_PrepareForPush_NilRepository(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + resources := []*model.Resource{ + { + Type: model.ResourceTypeImage, + Metadata: &model.ResourceMetadata{}, + }, + } + + err := a.PrepareForPush(resources) + assert.Error(t, err) +} + +func TestAdapter_PrepareForPush_NilRepositoryName(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + resources := []*model.Resource{ + { + Type: model.ResourceTypeImage, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{}, + }, + }, + } + + err := a.PrepareForPush(resources) + assert.Error(t, err) +} diff --git a/src/pkg/reg/adapter/volcenginecr/artifact_registry.go b/src/pkg/reg/adapter/volcenginecr/artifact_registry.go new file mode 100644 index 000000000..ff3d74f77 --- /dev/null +++ b/src/pkg/reg/adapter/volcenginecr/artifact_registry.go @@ -0,0 +1,205 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcenginecr + +import ( + "fmt" + "strings" + + "github.com/opencontainers/go-digest" + "github.com/volcengine/volcengine-go-sdk/service/cr" + + "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/reg/model" + "github.com/goharbor/harbor/src/pkg/reg/util" +) + +// DeleteManifest VolcCR will use our own openAPI to delete Manifest +func (a *adapter) DeleteManifest(repository, reference string) (err error) { + parts := strings.SplitN(repository, "/", 2) + if len(parts) != 2 { + return fmt.Errorf("VolcEngineCR only support repo in format /, but got: %s", repository) + } + log.Warningf("namespace=%s, repository=%s, tag=%s", parts[0], parts[1], reference) + + if _, err := digest.Parse(reference); err != nil { + // get digest + resp, err := a.volcCrClient.ListTags(&cr.ListTagsInput{ + Registry: a.registryName, + Namespace: &parts[0], + Repository: &parts[1], + Filter: &cr.FilterForListTagsInput{ + Names: []*string{ + &reference, + }, + }, + }) + if err != nil { + return err + } + if resp == nil || resp.TotalCount == nil { + return fmt.Errorf("[VolcEngineCR.DeleteManifest] ListTags resp nil") + } + if *resp.TotalCount == 0 { + return nil + } + if resp.Items[0] == nil { + return fmt.Errorf("[VolcEngineCR.DeleteManifest] ListTags resp nil") + } + reference = *resp.Items[0].Digest + } + // listCandidateTags based on digest + tags, err := a.listCandidateTags(parts[0], parts[1], reference) + if err != nil { + log.Errorf("DeleteManifest error :%v", err) + return err + } + // deleteTags + err = a.deleteTags(parts[0], parts[1], tags) + if err != nil { + log.Errorf("DeleteManifest error :%v", err) + } + return +} + +// DeleteTag VolcCR will use our own openAPI to delete tag +func (a *adapter) DeleteTag(repository, tag string) (err error) { + parts := strings.SplitN(repository, "/", 2) + if len(parts) != 2 { + return fmt.Errorf("VolcEngineCR only support repo in format /, but got: %s", repository) + } + log.Warningf("namespace=%s, repository=%s, tag=%s", parts[0], parts[1], tag) + + err = a.deleteTags(parts[0], parts[1], []*string{ + &tag, + }) + if err != nil { + log.Errorf("deleteTag error: %v", err) + } + return +} + +// FetchArtifacts VolcCR not support /v2/_catalog of Registry +func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, error) { + log.Debug("FetchArtifacts filters", "filters", filters) + // 1. get filter pattern + var repoPattern string + var tagsPattern string + for _, filter := range filters { + if filter.Type == model.FilterTypeName { + repoPattern = filter.Value.(string) + } + if filter.Type == model.FilterTypeTag { + tagsPattern = filter.Value.(string) + } + } + namespacePattern := strings.Split(repoPattern, "/")[0] + + log.Debug("read in filter patterns", "repoPattern", repoPattern, "tagsPattern", tagsPattern) + + // 2. list namespace candidtes + namespaces, err := a.listCandidateNamespaces(namespacePattern) + if err != nil { + log.Errorf("FetchArtifacts error: %v", err) + return nil, err + } + log.Debug("FetchArtifacts filtered namespace", "namespace", namespaces) + + // 3. list repos + var nsRepos []string + for _, ns := range namespaces { + repoCandidates, err := a.listRepositories(ns) + if err != nil { + log.Error("FetchArtifacts error", "error", err) + return nil, err + } + log.Debug(" FetchArtifacts list repo", "repos: ", repoCandidates) + for _, r := range repoCandidates { + nsRepoCandidate := fmt.Sprintf("%s/%s", ns, r) + ok, err := util.Match(repoPattern, nsRepoCandidate) + if err != nil { + log.Error("FetchArtifacts error", "error", err) + return nil, err + } + log.Debug("filter namespaced repository", "repoPattern: ", repoPattern, "repo: ", nsRepoCandidate) + if ok { + nsRepos = append(nsRepos, nsRepoCandidate) + } + } + } + log.Debug("filter namespaced repository", "length", len(nsRepos)) + + // 4. list tags + var rawResources = make([]*model.Resource, len(nsRepos)) + resources := make([]*model.Resource, 0) + runner := utils.NewLimitedConcurrentRunner(concurrentLimit) + + for idx, repo := range nsRepos { + i := idx + nsRepo := repo + runner.AddTask(func() error { + repoArr := strings.SplitN(nsRepo, "/", 2) + // note list tag don't tell different oci types now + candidateTags, err := a.listAllTags(repoArr[0], repoArr[1]) + if err != nil { + log.Error("fail to list all tags", "nsRepo", nsRepo) + return fmt.Errorf("volcengineCR fail to list all tags %w", err) + } + + tags := make([]string, 0) + if tagsPattern != "" { + for _, candidateTag := range candidateTags { + ok, err := util.Match(tagsPattern, candidateTag) + if err != nil { + return fmt.Errorf("fail to match tag pattern, error=%w", err) + } + if ok { + tags = append(tags, candidateTag) + } + } + } else { + tags = candidateTags + } + + log.Debug("filter tags") + + if len(tags) > 0 { + rawResources[i] = &model.Resource{ + Type: model.ResourceTypeImage, + Registry: a.registry, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: nsRepo, + }, + Vtags: tags, + }, + } + } + return nil + }) + } + if err = runner.Wait(); err != nil { + return nil, fmt.Errorf("failed to fetch artifacts: %w", err) + } + + for _, res := range rawResources { + if res != nil { + resources = append(resources, res) + } + } + + return resources, nil +} diff --git a/src/pkg/reg/adapter/volcenginecr/artifact_registry_test.go b/src/pkg/reg/adapter/volcenginecr/artifact_registry_test.go new file mode 100644 index 000000000..a9645ddc7 --- /dev/null +++ b/src/pkg/reg/adapter/volcenginecr/artifact_registry_test.go @@ -0,0 +1,55 @@ +package volcenginecr + +import ( + "github.com/goharbor/harbor/src/pkg/reg/model" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestArtifactRegistry_DeleteManifest(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + err := a.DeleteManifest("ut_test/ut_test", "sha256:7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc") + assert.Error(t, err) +} + +func TestArtifactRegistry_DeleteTag(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + err := a.DeleteTag("ut_test/ut_test", "v1") + assert.Error(t, err) +} + +func TestArtifactRegistry_FetchArtifacts(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + tests := []struct { + name string + filter model.Filter + wantErr bool + }{ + {"filter name", + model.Filter{ + Type: model.FilterTypeName, + Value: "ut_test", + }, + true}, + {"filter tag", + model.Filter{ + Type: model.FilterTypeTag, + Value: "v1", + }, + true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := a.FetchArtifacts([]*model.Filter{&tt.filter}) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/src/pkg/reg/adapter/volcenginecr/auth.go b/src/pkg/reg/adapter/volcenginecr/auth.go new file mode 100644 index 000000000..b0a08e924 --- /dev/null +++ b/src/pkg/reg/adapter/volcenginecr/auth.go @@ -0,0 +1,93 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcenginecr + +import ( + "errors" + "fmt" + "net/http" + "time" + + "github.com/volcengine/volcengine-go-sdk/service/cr" + + "github.com/goharbor/harbor/src/common/http/modifier" + "github.com/goharbor/harbor/src/lib/log" +) + +// Credential ... +type Credential modifier.Modifier + +type volcCredential struct { + client *cr.CR + registry string + authCache *authCache +} + +type authCache struct { + username string + password string + expireAt *time.Time +} + +var _ Credential = &volcCredential{} + +// NewAuth will get a temporary username and password via cr GetAuthorizationToken action for docker login +func NewAuth(client *cr.CR, registry string) Credential { + return &volcCredential{ + client: client, + registry: registry, + authCache: &authCache{}, + } +} + +func (c *volcCredential) Modify(r *http.Request) (err error) { + if c.client == nil { + return errNilVolcCrClient + } + if !c.isCacheAuthValid() { + log.Debugf("update token %s\n", r.Host) + authResp, err := c.client.GetAuthorizationToken(&cr.GetAuthorizationTokenInput{ + Registry: &c.registry, + }) + if err != nil { + return err + } + if authResp == nil || authResp.Username == nil || authResp.Token == nil || authResp.ExpireTime == nil { + return errors.New("[VolcengineCR] GetAuthorizationToken output nil") + } + c.authCache.username = *authResp.Username + c.authCache.password = *authResp.Token + expireTime, err := time.Parse(time.RFC3339, *authResp.ExpireTime) + if err != nil { + log.Errorf("fail to parse expire time returned: %v", err) + return fmt.Errorf("[VolcengineCR] fail to parse expire time returned: %v", err) + } + c.authCache.expireAt = &expireTime + } else { + log.Debug("token cached") + } + r.SetBasicAuth(c.authCache.username, c.authCache.password) + return nil +} + +func (c *volcCredential) isCacheAuthValid() bool { + if c.authCache == nil || c.authCache.expireAt == nil { + return false + } + if time.Now().After(*c.authCache.expireAt) { + return false + } + return true +} diff --git a/src/pkg/reg/adapter/volcenginecr/auth_test.go b/src/pkg/reg/adapter/volcenginecr/auth_test.go new file mode 100644 index 000000000..6af5566f6 --- /dev/null +++ b/src/pkg/reg/adapter/volcenginecr/auth_test.go @@ -0,0 +1,29 @@ +package volcenginecr + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + volcCR "github.com/volcengine/volcengine-go-sdk/service/cr" + "github.com/volcengine/volcengine-go-sdk/volcengine" + "github.com/volcengine/volcengine-go-sdk/volcengine/credentials" + volcSession "github.com/volcengine/volcengine-go-sdk/volcengine/session" +) + +func Test_Modify_nilCR(t *testing.T) { + c := &volcCredential{} + err := c.Modify(&http.Request{}) + assert.Error(t, err) +} + +func Test_Modify(t *testing.T) { + config := volcengine.NewConfig(). + WithCredentials(credentials.NewStaticCredentials("", "", "")). + WithRegion("cn-beijing") + sess, _ := volcSession.NewSession(config) + client := volcCR.New(sess) + c := &volcCredential{client: client} + err := c.Modify(&http.Request{}) + assert.Error(t, err) +} diff --git a/src/pkg/reg/adapter/volcenginecr/consts.go b/src/pkg/reg/adapter/volcenginecr/consts.go new file mode 100644 index 000000000..d519d2bb0 --- /dev/null +++ b/src/pkg/reg/adapter/volcenginecr/consts.go @@ -0,0 +1,33 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcenginecr + +import "errors" + +var ( + regionRegs = map[string]interface{}{ + "(.*)-(cn-.*)": nil, + } + errListNamespaceResp = errors.New("[VolcengineCR adapt] ListNamespaces resp nil") + errListRepositoriesResp = errors.New("[VolcengineCR adapt] ListRepositories resp nil") + errListTagsResp = errors.New("[VolcengineCR adapt] ListTags resp nil") + errPareseDigest = errors.New("[VolcengineCR adapt] fail to parse reference") + errNilVolcCrClient = errors.New("[volcengine-cr.createRepository] nil volcCr client") +) + +const ( + MaxPageSize int64 = 100 + concurrentLimit int = 3 +) diff --git a/src/pkg/reg/adapter/volcenginecr/helper.go b/src/pkg/reg/adapter/volcenginecr/helper.go new file mode 100644 index 000000000..8af780f31 --- /dev/null +++ b/src/pkg/reg/adapter/volcenginecr/helper.go @@ -0,0 +1,66 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcenginecr + +import ( + "errors" + "fmt" + "net/http" + "regexp" + + "github.com/docker/distribution/registry/client/auth/challenge" + + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/reg/util" +) + +func getRegionRegistryName(url string) (string, string, error) { + reg := regexp.MustCompile(`https://(.*)\.cr\.volces|ivolces\.com`) + rs := reg.FindStringSubmatch(url) + if rs == nil || len(rs) != 2 { + return "", "", errors.New("Invalid url") + } + registryNameRegion := rs[1] + for regionReg := range regionRegs { + reg = regexp.MustCompile(regionReg) + res := reg.FindStringSubmatch(registryNameRegion) + if res == nil || len(res) != 3 { + log.Debug("fail to match", "reg", regionReg) + continue + } + return res[2], res[1], nil + } + + return "", "", errors.New("invalid region") +} + +func getRealmService(host string, insecure bool) (string, string, error) { + client := &http.Client{ + Transport: util.GetHTTPTransport(insecure), + } + + resp, err := client.Get(host + "/v2/") + if err != nil { + return "", "", err + } + defer resp.Body.Close() // nolint + challenges := challenge.ResponseChallenges(resp) + for _, challenge := range challenges { + if challenge.Scheme == "bearer" { + return challenge.Parameters["realm"], challenge.Parameters["service"], nil + } + } + return "", "", fmt.Errorf("bearer auth scheme isn't supported: %v", challenges) +} diff --git a/src/pkg/reg/adapter/volcenginecr/helper_test.go b/src/pkg/reg/adapter/volcenginecr/helper_test.go new file mode 100644 index 000000000..e08f72797 --- /dev/null +++ b/src/pkg/reg/adapter/volcenginecr/helper_test.go @@ -0,0 +1,56 @@ +package volcenginecr + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_getRegionRegistryNamer(t *testing.T) { + tests := []struct { + name string + url string + wantRegion string + wantRegistry string + wantErr bool + }{ + {"registry beijing", "https://enterprise-cn-beijing.cr.volces.com", "cn-beijing", "enterprise", false}, + {"invalid url", "http://enterprise-cn-beijing.cr.volces.com", "", "", true}, + {"invalid region", "https://enterprise-us-test.cr.volces.com", "", "", true}, + {"invalid suffix", "https://enterprise-us-test.cr-test.volces.com", "", "", true}, + {"registry shanghai", "https://cn-beijing-cn-shanghai.cr.volces.com", "cn-shanghai", "cn-beijing", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotRegion, gotRegistry, err := getRegionRegistryName(tt.url) + if tt.wantErr { + assert.NotNil(t, err) + } + assert.Equal(t, tt.wantRegion, gotRegion) + assert.Equal(t, tt.wantRegistry, gotRegistry) + }) + } +} + +func Test_getRealmService(t *testing.T) { + tests := []struct { + name string + host string + insecure bool + wantErr bool + }{ + {"ping success", "https://cr-cn-beijing.volces.com", false, false}, + {"ping success", "https://cr-cn-beijing.volces.com", true, false}, + {"ping error", "https://cr-test-cn-beijing.volces.com", true, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, _, err := getRealmService(tt.host, tt.insecure) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/src/pkg/reg/adapter/volcenginecr/volccr.go b/src/pkg/reg/adapter/volcenginecr/volccr.go new file mode 100644 index 000000000..753cfe599 --- /dev/null +++ b/src/pkg/reg/adapter/volcenginecr/volccr.go @@ -0,0 +1,334 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcenginecr + +import ( + "math" + + "github.com/opencontainers/go-digest" + "github.com/volcengine/volcengine-go-sdk/service/cr" + + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/reg/util" +) + +func (a *adapter) createNamespace(namespace string) (err error) { + if a.volcCrClient == nil { + return errNilVolcCrClient + } + // check if exists + exist, err := a.namespaceExist(namespace) + if err != nil { + return + } + // if exists; skip create + if exist { + return nil + } + + // create Namespace + _, err = a.volcCrClient.CreateNamespace(&cr.CreateNamespaceInput{ + Registry: a.registryName, + Name: &namespace, + }) + if err != nil { + log.Debugf("CreateNamespace error:%v", err) + return err + } + return nil +} + +func (a *adapter) createRepository(namespace, repository string) (err error) { + if a.volcCrClient == nil { + return errNilVolcCrClient + } + // check if exists + res, err := a.volcCrClient.ListRepositories(&cr.ListRepositoriesInput{ + Registry: a.registryName, + Filter: &cr.FilterForListRepositoriesInput{ + Names: []*string{ + &repository, + }, + Namespaces: []*string{ + &namespace, + }, + }, + }) + if err != nil { + log.Debugf("ListRepositories error:%v", err) + return err + } + // if exists; skip create + if res != nil && res.TotalCount != nil && *res.TotalCount > 0 { + return nil + } + + // create Repository + _, err = a.volcCrClient.CreateRepository(&cr.CreateRepositoryInput{ + Registry: a.registryName, + Namespace: &namespace, + Name: &repository, + }) + if err != nil { + log.Debugf("CreateRepository error:%v", err) + return err + } + return nil +} + +func (a *adapter) deleteTags(namespace, repository string, tags []*string) error { + _, err := a.volcCrClient.DeleteTags(&cr.DeleteTagsInput{ + Registry: a.registryName, + Namespace: &namespace, + Repository: &repository, + Names: tags, + }) + return err +} + +func (a *adapter) listCandidateNamespaces(namespacePattern string) ([]string, error) { + if a.volcCrClient == nil { + return []string{}, errNilVolcCrClient + } + namespaces := make([]string, 0) + // filter namespaces + if len(namespacePattern) > 0 { + if nms, ok := util.IsSpecificPathComponent(namespacePattern); ok { + // Check if namespace exist + for _, ns := range nms { + exist, err := a.namespaceExist(ns) + if err != nil { + return nil, err + } + if !exist { + continue + } + namespaces = append(namespaces, nms...) + } + } + } + + if len(namespaces) > 0 { + log.Debug("list candidate namespace", "pattern", namespacePattern, "namespaces", namespaces) + return namespaces, nil + } + + // list all + return a.listNamespaces() +} + +func (a *adapter) listNamespaces() ([]string, error) { + if a.volcCrClient == nil { + return []string{}, errNilVolcCrClient + } + pageSize := MaxPageSize + pageNumber := int64(1) + initCondition := true + var remain int64 = math.MaxInt64 + var nsList []string + + for remain > 0 { + resp, err := a.volcCrClient.ListNamespaces( + &cr.ListNamespacesInput{ + Registry: a.registryName, + PageSize: &pageSize, + PageNumber: &pageNumber, + }) + + if err != nil { + return nil, err + } + if resp == nil || resp.TotalCount == nil { + return nil, errListNamespaceResp + } + if initCondition { + nsList = make([]string, 0, *resp.TotalCount) + remain = *resp.TotalCount - pageSize + initCondition = false + } else { + remain -= pageSize + } + // be careful with state machine. + pageNumber++ + for _, nsInfo := range resp.Items { + if nsInfo != nil && nsInfo.Name != nil { + nsList = append(nsList, *nsInfo.Name) + } + } + } + return nsList, nil +} + +func (a *adapter) listRepositories(namespace string) ([]string, error) { + if a.volcCrClient == nil { + return []string{}, errNilVolcCrClient + } + pageSize := MaxPageSize + pageNumber := int64(1) + initCondition := true + var remain int64 = math.MaxInt64 + var repoList []string + + for remain > 0 { + resp, err := a.volcCrClient.ListRepositories( + &cr.ListRepositoriesInput{ + Registry: a.registryName, + PageSize: &pageSize, + PageNumber: &pageNumber, + Filter: &cr.FilterForListRepositoriesInput{ + Namespaces: []*string{&namespace}, + }, + }) + if err != nil { + return nil, err + } + if resp == nil || resp.TotalCount == nil { + return nil, errListRepositoriesResp + } + if initCondition { + repoList = make([]string, 0, *resp.TotalCount) + remain = *resp.TotalCount - pageSize + initCondition = false + } else { + remain -= pageSize + } + // be careful with state machine. + pageNumber++ + for _, repoInfo := range resp.Items { + if repoInfo != nil && repoInfo.Name != nil { + repoList = append(repoList, *repoInfo.Name) + } + } + } + return repoList, nil +} + +// listAllTags list all tags of different artifacts with given namespace and repo +func (a *adapter) listAllTags(namespace, repo string) ([]string, error) { + if a.volcCrClient == nil { + return []string{}, errNilVolcCrClient + } + pageSize := MaxPageSize + pageNumber := int64(1) + initCondition := true + var remain int64 = math.MaxInt64 + var tagList []string + + for remain > 0 { + resp, err := a.volcCrClient.ListTags( + &cr.ListTagsInput{ + Registry: a.registryName, + Namespace: &namespace, + Repository: &repo, + PageSize: &pageSize, + PageNumber: &pageNumber, + }) + if err != nil { + return nil, err + } + if resp == nil || resp.TotalCount == nil { + return nil, errListTagsResp + } + if initCondition { + tagList = make([]string, 0, *resp.TotalCount) + remain = *resp.TotalCount - pageSize + initCondition = false + } else { + remain -= pageSize + } + pageNumber++ + for _, tagInfo := range resp.Items { + tagList = append(tagList, *tagInfo.Name) + } + } + return tagList, nil +} + +func (a *adapter) listCandidateTags(namespace, repository, reference string) ([]*string, error) { + if a.volcCrClient == nil { + return []*string{}, errNilVolcCrClient + } + pageSize := MaxPageSize + pageNumber := int64(1) + initCondition := true + var remain int64 = math.MaxInt64 + var tagList []*string + desiredDig, err := digest.Parse(reference) + if err != nil { + return tagList, errPareseDigest + } + + for remain > 0 { + resp, err := a.volcCrClient.ListTags( + &cr.ListTagsInput{ + Registry: a.registryName, + Namespace: &namespace, + Repository: &repository, + PageSize: &pageSize, + PageNumber: &pageNumber, + }) + if err != nil { + return nil, err + } + if resp == nil || resp.TotalCount == nil { + return nil, errPareseDigest + } + if initCondition { + tagList = make([]*string, 0, *resp.TotalCount) + remain = *resp.TotalCount - pageSize + initCondition = false + } else { + remain -= pageSize + } + pageNumber++ + for _, tagInfo := range resp.Items { + if tagInfo != nil && tagInfo.Name != nil && tagInfo.Digest != nil { + dig, err := digest.Parse(*tagInfo.Digest) + if err != nil { + log.Debug("fail to parase digest", "tag", tagInfo) + continue + } + if desiredDig.String() == dig.String() { + tagList = append(tagList, tagInfo.Name) + } + } + } + } + return tagList, nil +} + +func (a *adapter) namespaceExist(namespace string) (bool, error) { + if a.volcCrClient == nil { + return false, errNilVolcCrClient + } + resp, err := a.volcCrClient.ListNamespaces(&cr.ListNamespacesInput{ + Registry: a.registryName, + Filter: &cr.FilterForListNamespacesInput{ + Names: []*string{ + &namespace, + }, + }, + }) + if err != nil { + return false, err + } + if resp == nil || resp.TotalCount == nil { + return false, errListNamespaceResp + } + if *resp.TotalCount > 0 { + return true, nil + } + return false, nil +} diff --git a/src/pkg/reg/adapter/volcenginecr/volccr_test.go b/src/pkg/reg/adapter/volcenginecr/volccr_test.go new file mode 100644 index 000000000..961df39eb --- /dev/null +++ b/src/pkg/reg/adapter/volcenginecr/volccr_test.go @@ -0,0 +1,70 @@ +package volcenginecr + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestVolccr_createNamespace(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + err := a.createNamespace("ut_test") + assert.Error(t, err) +} + +func TestVolccr_createRepository(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + err := a.createRepository("ut_test", "ut_test") + assert.Error(t, err) +} + +func TestVolccr_deleteTags(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + err := a.deleteTags("ut_test", "ut_test", []*string{}) + assert.Error(t, err) +} + +func TestVolccr_listCandidateNamespaces(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + _, err := a.listCandidateNamespaces("ut_test") + assert.Error(t, err) +} + +func TestVolccr_listNamespaces(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + _, err := a.listNamespaces() + assert.Error(t, err) +} + +func TestVolccr_listRepositories(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + _, err := a.listRepositories("ut_test") + assert.Error(t, err) +} + +func TestVolccr_listAllTags(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + _, err := a.listAllTags("ut_test", "ut_test") + assert.Error(t, err) +} + +func TestVolccr_listCandidateTags(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + _, err := a.listCandidateTags("ut_test", "ut_test", "sha256:7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc") + assert.Error(t, err) +} + +func TestVolccr_namespaceExist(t *testing.T) { + a, s := getMockAdapter_withoutCred(t, true, true) + defer s.Close() + _, err := a.namespaceExist("ut_test") + assert.Error(t, err) +} diff --git a/src/pkg/reg/manager.go b/src/pkg/reg/manager.go index 251aec47b..c1d37671d 100644 --- a/src/pkg/reg/manager.go +++ b/src/pkg/reg/manager.go @@ -22,6 +22,8 @@ import ( "github.com/goharbor/harbor/src/lib/config" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/reg/adapter" + "github.com/goharbor/harbor/src/pkg/reg/dao" + "github.com/goharbor/harbor/src/pkg/reg/model" // register the AliACR adapter _ "github.com/goharbor/harbor/src/pkg/reg/adapter/aliacr" @@ -51,8 +53,8 @@ import ( _ "github.com/goharbor/harbor/src/pkg/reg/adapter/quay" // register the TencentCloud TCR adapter _ "github.com/goharbor/harbor/src/pkg/reg/adapter/tencentcr" - "github.com/goharbor/harbor/src/pkg/reg/dao" - "github.com/goharbor/harbor/src/pkg/reg/model" + // register the VolcEngine CR Registry adapter + _ "github.com/goharbor/harbor/src/pkg/reg/adapter/volcenginecr" ) var ( diff --git a/src/pkg/reg/model/registry.go b/src/pkg/reg/model/registry.go index 6c8a09cc4..1b4f945d7 100644 --- a/src/pkg/reg/model/registry.go +++ b/src/pkg/reg/model/registry.go @@ -34,6 +34,7 @@ const ( RegistryTypeDTR = "dtr" RegistryTypeTencentTcr = "tencent-tcr" RegistryTypeGithubCR = "github-ghcr" + RegistryTypeVolcCR = "volcengine-cr" RegistryTypeHelmHub = "helm-hub" RegistryTypeArtifactHub = "artifact-hub" diff --git a/src/portal/src/app/shared/services/endpoint.service.ts b/src/portal/src/app/shared/services/endpoint.service.ts index a22a1d6fd..0b007bca0 100644 --- a/src/portal/src/app/shared/services/endpoint.service.ts +++ b/src/portal/src/app/shared/services/endpoint.service.ts @@ -29,6 +29,7 @@ export const ADAPTERS_MAP = { dtr: 'DTR', 'tencent-tcr': 'Tencent TCR', 'github-ghcr': 'Github GHCR', + 'volcengine-cr': 'VolcEngine CR', }; /**