From 8f11cb7ff09ba552734a3fc062524cfd2f778aec Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Sat, 4 Apr 2020 20:59:46 +0800 Subject: [PATCH] Support replication between Harbor 2.0 and 1.x Fixes #11374, fixes #11302, support replication between Harbor 2.0 and 1.x by providing versioning adapter Signed-off-by: Wenkai Yin --- src/replication/adapter/harbor/adaper.go | 54 ++++ .../adapter/harbor/{ => base}/adapter.go | 264 ++++++++---------- .../adapter/harbor/{ => base}/adapter_test.go | 41 +-- .../harbor/{ => base}/chart_registry.go | 42 +-- .../harbor/{ => base}/chart_registry_test.go | 16 +- src/replication/adapter/harbor/base/client.go | 135 +++++++++ .../adapter/harbor/image_registry.go | 177 ------------ .../adapter/harbor/image_registry_test.go | 109 -------- src/replication/adapter/harbor/v1/adapter.go | 126 +++++++++ src/replication/adapter/harbor/v1/client.go | 73 +++++ src/replication/adapter/harbor/v2/adapter.go | 134 +++++++++ src/replication/adapter/harbor/v2/client.go | 80 ++++++ src/server/route.go | 5 + src/server/v2.0/route/legacy.go | 1 + src/server/v2.0/route/route.go | 7 +- src/server/version.go | 39 +++ 16 files changed, 819 insertions(+), 484 deletions(-) create mode 100644 src/replication/adapter/harbor/adaper.go rename src/replication/adapter/harbor/{ => base}/adapter.go (52%) rename src/replication/adapter/harbor/{ => base}/adapter_test.go (89%) rename src/replication/adapter/harbor/{ => base}/chart_registry.go (83%) rename src/replication/adapter/harbor/{ => base}/chart_registry_test.go (93%) create mode 100644 src/replication/adapter/harbor/base/client.go delete mode 100644 src/replication/adapter/harbor/image_registry.go delete mode 100644 src/replication/adapter/harbor/image_registry_test.go create mode 100644 src/replication/adapter/harbor/v1/adapter.go create mode 100644 src/replication/adapter/harbor/v1/client.go create mode 100644 src/replication/adapter/harbor/v2/adapter.go create mode 100644 src/replication/adapter/harbor/v2/client.go create mode 100644 src/server/version.go diff --git a/src/replication/adapter/harbor/adaper.go b/src/replication/adapter/harbor/adaper.go new file mode 100644 index 000000000..405017437 --- /dev/null +++ b/src/replication/adapter/harbor/adaper.go @@ -0,0 +1,54 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package harbor + +import ( + "github.com/goharbor/harbor/src/lib/log" + adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/adapter/harbor/base" + v1 "github.com/goharbor/harbor/src/replication/adapter/harbor/v1" + v2 "github.com/goharbor/harbor/src/replication/adapter/harbor/v2" + "github.com/goharbor/harbor/src/replication/model" +) + +func init() { + if err := adp.RegisterFactory(model.RegistryTypeHarbor, new(factory)); err != nil { + log.Errorf("failed to register factory for %s: %v", model.RegistryTypeHarbor, err) + return + } + log.Infof("the factory for adapter %s registered", model.RegistryTypeHarbor) +} + +type factory struct { +} + +func (f *factory) Create(r *model.Registry) (adp.Adapter, error) { + base, err := base.New(r) + if err != nil { + return nil, err + } + version := base.GetAPIVersion() + // no API version, it's instance of Harbor 1.x + if len(version) == 0 { + log.Debug("no API version, create the v1 adapter") + return v1.New(base), nil + } + log.Debugf("API version is %s, create the v2 adapter", version) + return v2.New(base), nil +} + +func (f *factory) AdapterPattern() *model.AdapterPattern { + return nil +} diff --git a/src/replication/adapter/harbor/adapter.go b/src/replication/adapter/harbor/base/adapter.go similarity index 52% rename from src/replication/adapter/harbor/adapter.go rename to src/replication/adapter/harbor/base/adapter.go index 5471fa9b5..f3d66fd51 100644 --- a/src/replication/adapter/harbor/adapter.go +++ b/src/replication/adapter/harbor/base/adapter.go @@ -12,81 +12,48 @@ // See the License for the specific language governing permissions and // limitations under the License. -package harbor +package base import ( "errors" - "fmt" "net/http" "strconv" "strings" - "github.com/goharbor/harbor/src/common/api" common_http "github.com/goharbor/harbor/src/common/http" "github.com/goharbor/harbor/src/common/http/modifier" common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth" - "github.com/goharbor/harbor/src/jobservice/config" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/registry/auth/basic" - - adp "github.com/goharbor/harbor/src/replication/adapter" "github.com/goharbor/harbor/src/replication/adapter/native" "github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/util" ) -func init() { - if err := adp.RegisterFactory(model.RegistryTypeHarbor, new(factory)); err != nil { - log.Errorf("failed to register factory for %s: %v", model.RegistryTypeHarbor, err) - return - } - log.Infof("the factory for adapter %s registered", model.RegistryTypeHarbor) -} - -type factory struct { -} - -// Create ... -func (f *factory) Create(r *model.Registry) (adp.Adapter, error) { - return newAdapter(r) -} - -// AdapterPattern ... -func (f *factory) AdapterPattern() *model.AdapterPattern { - return nil -} - -var ( - _ adp.Adapter = (*adapter)(nil) - _ adp.ArtifactRegistry = (*adapter)(nil) - _ adp.ChartRegistry = (*adapter)(nil) -) - -type adapter struct { - *native.Adapter - registry *model.Registry - url string - client *common_http.Client -} - -func newAdapter(registry *model.Registry) (*adapter, error) { - var transport *http.Transport - if registry.URL == config.GetCoreURL() { - transport = common_http.GetHTTPTransport(common_http.SecureTransport) - } else { - transport = util.GetHTTPTransport(registry.Insecure) - } - // local Harbor instance - if registry.Credential != nil && registry.Credential.Type == model.CredentialTypeSecret { +// New creates an instance of the base adapter +func New(registry *model.Registry) (*Adapter, error) { + if isLocalHarbor(registry) { + // when the adapter is created for local Harbor, returns the "http://127.0.0.1:8080" + // as URL to avoid issue https://github.com/goharbor/harbor-helm/issues/222 + // when harbor is deployed on Kubernetes + url := "http://127.0.0.1:8080" + if common_http.InternalTLSEnabled() { + url = "https://127.0.0.1:8443" + } authorizer := common_http_auth.NewSecretAuthorizer(registry.Credential.AccessSecret) - return &adapter{ - registry: registry, - url: registry.URL, - client: common_http.NewClient( - &http.Client{ - Transport: transport, - }, authorizer), - Adapter: native.NewAdapterWithAuthorizer(registry, authorizer), + httpClient := common_http.NewClient(&http.Client{ + Transport: common_http.GetHTTPTransport(common_http.SecureTransport), + }, authorizer) + client, err := NewClient(url, httpClient) + if err != nil { + return nil, err + } + return &Adapter{ + Adapter: native.NewAdapterWithAuthorizer(registry, authorizer), + Registry: registry, + Client: client, + url: url, + httpClient: httpClient, }, nil } @@ -96,22 +63,43 @@ func newAdapter(registry *model.Registry) (*adapter, error) { registry.Credential.AccessKey, registry.Credential.AccessSecret)) } - return &adapter{ - registry: registry, - url: registry.URL, - client: common_http.NewClient( - &http.Client{ - Transport: transport, - }, authorizers...), - Adapter: native.NewAdapter(registry), + httpClient := common_http.NewClient(&http.Client{ + Transport: common_http.GetHTTPTransportByInsecure(registry.Insecure), + }, authorizers...) + client, err := NewClient(registry.URL, httpClient) + if err != nil { + return nil, err + } + return &Adapter{ + Adapter: native.NewAdapter(registry), + Registry: registry, + Client: client, + url: registry.URL, + httpClient: httpClient, }, nil } -func (a *adapter) Info() (*model.RegistryInfo, error) { +// Adapter is the base adapter for Harbor +type Adapter struct { + *native.Adapter + Registry *model.Registry + Client *Client + + // url and httpClient can be removed if we don't support replicate chartmuseum charts anymore + url string + httpClient *common_http.Client +} + +// GetAPIVersion returns the supported API version of the Harbor instance that the adapter is created for +func (a *Adapter) GetAPIVersion() string { + return a.Client.APIVersion +} + +// Info provides the information of the Harbor registry instance +func (a *Adapter) Info() (*model.RegistryInfo, error) { info := &model.RegistryInfo{ Type: model.RegistryTypeHarbor, SupportedResourceTypes: []model.ResourceType{ - model.ResourceTypeArtifact, model.ResourceTypeImage, }, SupportedResourceFilters: []*model.FilterStyle{ @@ -130,40 +118,31 @@ func (a *adapter) Info() (*model.RegistryInfo, error) { }, } - sys := &struct { - ChartRegistryEnabled bool `json:"with_chartmuseum"` - }{} - if err := a.client.Get(fmt.Sprintf("%s/api/%s/systeminfo", a.getURL(), api.APIVersion), sys); err != nil { + enabled, err := a.Client.ChartRegistryEnabled() + if err != nil { return nil, err } - if sys.ChartRegistryEnabled { + if enabled { info.SupportedResourceTypes = append(info.SupportedResourceTypes, model.ResourceTypeChart) } - labels := []*struct { - Name string `json:"name"` - }{} - // label isn't supported in some previous version of Harbor - if err := a.client.Get(fmt.Sprintf("%s/api/%s/labels?scope=g", a.getURL(), api.APIVersion), &labels); err != nil { - if e, ok := err.(*common_http.Error); !ok || e.Code != http.StatusNotFound { - return nil, err - } - } else { - ls := []string{} - for _, label := range labels { - ls = append(ls, label.Name) - } - labelFilter := &model.FilterStyle{ + + labels, err := a.Client.ListLabels() + if err != nil { + return nil, err + } + info.SupportedResourceFilters = append(info.SupportedResourceFilters, + &model.FilterStyle{ Type: model.FilterTypeLabel, Style: model.FilterStyleTypeList, - Values: ls, - } - info.SupportedResourceFilters = append(info.SupportedResourceFilters, labelFilter) - } + Values: labels, + }) + return info, nil } -func (a *adapter) PrepareForPush(resources []*model.Resource) error { - projects := map[string]*project{} +// PrepareForPush creates projects +func (a *Adapter) PrepareForPush(resources []*model.Resource) error { + projects := map[string]*Project{} for _, resource := range resources { if resource == nil { return errors.New("the resource cannot be null") @@ -186,21 +165,13 @@ func (a *adapter) PrepareForPush(resources []*model.Resource) error { if exist { metadata = mergeMetadata(pro.Metadata, metadata) } - projects[projectName] = &project{ + projects[projectName] = &Project{ Name: projectName, Metadata: metadata, } } for _, project := range projects { - pro := struct { - Name string `json:"project_name"` - Metadata map[string]interface{} `json:"metadata"` - }{ - Name: project.Name, - Metadata: project.Metadata, - } - err := a.client.Post(fmt.Sprintf("%s/api/%s/projects", a.getURL(), api.APIVersion), pro) - if err != nil { + if err := a.Client.CreateProject(project.Name, project.Metadata); err != nil { if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict { log.Debugf("got 409 when trying to create project %s", project.Name) continue @@ -212,6 +183,44 @@ func (a *adapter) PrepareForPush(resources []*model.Resource) error { return nil } +// ListProjects lists projects +func (a *Adapter) ListProjects(filters []*model.Filter) ([]*Project, error) { + pattern := "" + for _, filter := range filters { + if filter.Type == model.FilterTypeName { + pattern = filter.Value.(string) + break + } + } + var projects []*Project + if len(pattern) > 0 { + substrings := strings.Split(pattern, "/") + projectPattern := substrings[0] + names, ok := util.IsSpecificPathComponent(projectPattern) + if ok { + for _, name := range names { + project, err := a.Client.GetProject(name) + if err != nil { + return nil, err + } + if project == nil { + continue + } + projects = append(projects, project) + } + } + } + if len(projects) > 0 { + var names []string + for _, project := range projects { + names = append(names, project.Name) + } + log.Debugf("parsed the projects %v from pattern %s", names, pattern) + return projects, nil + } + return a.Client.ListProjects("") +} + func abstractPublicMetadata(metadata map[string]interface{}) map[string]interface{} { if metadata == nil { return nil @@ -257,56 +266,13 @@ func parsePublic(metadata map[string]interface{}) bool { return false } -type project struct { +// Project model +type Project struct { ID int64 `json:"project_id"` Name string `json:"name"` Metadata map[string]interface{} `json:"metadata"` } -func (a *adapter) getProjects(name string) ([]*project, error) { - projects := []*project{} - url := fmt.Sprintf("%s/api/%s/projects?name=%s&page=1&page_size=500", a.getURL(), api.APIVersion, name) - if err := a.client.GetAndIteratePagination(url, &projects); err != nil { - return nil, err - } - return projects, nil -} - -func (a *adapter) getProject(name string) (*project, error) { - // TODO need an API to exact match project by name - projects, err := a.getProjects(name) - if err != nil { - return nil, err - } - - for _, pro := range projects { - if pro.Name == name { - p := &project{ - ID: pro.ID, - Name: name, - } - if pro.Metadata != nil { - metadata := map[string]interface{}{} - for key, value := range pro.Metadata { - metadata[key] = value - } - p.Metadata = metadata - } - return p, nil - } - } - return nil, nil -} - -// when the adapter is created for local Harbor, returns the "http://127.0.0.1:8080" -// as URL to avoid issue https://github.com/goharbor/harbor-helm/issues/222 -// when harbor is deployed on Kubernetes -func (a *adapter) getURL() string { - if a.registry.Type == model.RegistryTypeHarbor && a.registry.Name == "Local" { - if common_http.InternalTLSEnabled() { - return "https://core:8443" - } - return "http://127.0.0.1:8080" - } - return a.url +func isLocalHarbor(registry *model.Registry) bool { + return registry.Type == model.RegistryTypeHarbor && registry.Name == "Local" } diff --git a/src/replication/adapter/harbor/adapter_test.go b/src/replication/adapter/harbor/base/adapter_test.go similarity index 89% rename from src/replication/adapter/harbor/adapter_test.go rename to src/replication/adapter/harbor/base/adapter_test.go index f553bdb2d..3027e0949 100644 --- a/src/replication/adapter/harbor/adapter_test.go +++ b/src/replication/adapter/harbor/base/adapter_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package harbor +package base import ( "net/http" @@ -25,11 +25,18 @@ import ( "github.com/stretchr/testify/require" ) +func TestGetAPIVersion(t *testing.T) { + adapter := &Adapter{ + Client: &Client{APIVersion: "1.0"}, + } + assert.Equal(t, "1.0", adapter.GetAPIVersion()) +} + func TestInfo(t *testing.T) { // chart museum enabled server := test.NewServer(&test.RequestHandlerMapping{ Method: http.MethodGet, - Pattern: "/api/v2.0/systeminfo", + Pattern: "/api/systeminfo", Handler: func(w http.ResponseWriter, r *http.Request) { data := `{"with_chartmuseum":true}` w.Write([]byte(data)) @@ -38,23 +45,22 @@ func TestInfo(t *testing.T) { registry := &model.Registry{ URL: server.URL, } - adapter, err := newAdapter(registry) + adapter, err := New(registry) require.Nil(t, err) info, err := adapter.Info() require.Nil(t, err) assert.Equal(t, model.RegistryTypeHarbor, info.Type) - assert.Equal(t, 2, len(info.SupportedResourceFilters)) + assert.Equal(t, 3, len(info.SupportedResourceFilters)) assert.Equal(t, 2, len(info.SupportedTriggers)) - assert.Equal(t, 3, len(info.SupportedResourceTypes)) - assert.Equal(t, model.ResourceTypeArtifact, info.SupportedResourceTypes[0]) - assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[1]) - assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[2]) + assert.Equal(t, 2, len(info.SupportedResourceTypes)) + assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0]) + assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[1]) server.Close() // chart museum disabled server = test.NewServer(&test.RequestHandlerMapping{ Method: http.MethodGet, - Pattern: "/api/v2.0/systeminfo", + Pattern: "/api/systeminfo", Handler: func(w http.ResponseWriter, r *http.Request) { data := `{"with_chartmuseum":false}` w.Write([]byte(data)) @@ -63,23 +69,22 @@ func TestInfo(t *testing.T) { registry = &model.Registry{ URL: server.URL, } - adapter, err = newAdapter(registry) + adapter, err = New(registry) require.Nil(t, err) info, err = adapter.Info() require.Nil(t, err) assert.Equal(t, model.RegistryTypeHarbor, info.Type) - assert.Equal(t, 2, len(info.SupportedResourceFilters)) + assert.Equal(t, 3, len(info.SupportedResourceFilters)) assert.Equal(t, 2, len(info.SupportedTriggers)) - assert.Equal(t, 2, len(info.SupportedResourceTypes)) - assert.Equal(t, model.ResourceTypeArtifact, info.SupportedResourceTypes[0]) - assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[1]) + assert.Equal(t, 1, len(info.SupportedResourceTypes)) + assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0]) server.Close() } func TestPrepareForPush(t *testing.T) { server := test.NewServer(&test.RequestHandlerMapping{ Method: http.MethodPost, - Pattern: "/api/v2.0/projects", + Pattern: "/api/projects", Handler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) }, @@ -87,7 +92,7 @@ func TestPrepareForPush(t *testing.T) { registry := &model.Registry{ URL: server.URL, } - adapter, err := newAdapter(registry) + adapter, err := New(registry) require.Nil(t, err) // nil resource err = adapter.PrepareForPush([]*model.Resource{nil}) @@ -133,7 +138,7 @@ func TestPrepareForPush(t *testing.T) { // project already exists server = test.NewServer(&test.RequestHandlerMapping{ Method: http.MethodPost, - Pattern: "/api/v2.0/projects", + Pattern: "/api/projects", Handler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusConflict) }, @@ -141,7 +146,7 @@ func TestPrepareForPush(t *testing.T) { registry = &model.Registry{ URL: server.URL, } - adapter, err = newAdapter(registry) + adapter, err = New(registry) require.Nil(t, err) err = adapter.PrepareForPush( []*model.Resource{ diff --git a/src/replication/adapter/harbor/chart_registry.go b/src/replication/adapter/harbor/base/chart_registry.go similarity index 83% rename from src/replication/adapter/harbor/chart_registry.go rename to src/replication/adapter/harbor/base/chart_registry.go index ee0a8ccb1..28ffdacd4 100644 --- a/src/replication/adapter/harbor/chart_registry.go +++ b/src/replication/adapter/harbor/base/chart_registry.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package harbor +package base import ( "bytes" @@ -23,9 +23,8 @@ import ( "net/http" "strings" - "github.com/goharbor/harbor/src/replication/filter" - common_http "github.com/goharbor/harbor/src/common/http" + "github.com/goharbor/harbor/src/replication/filter" "github.com/goharbor/harbor/src/replication/model" ) @@ -46,17 +45,18 @@ type chartVersionMetadata struct { URLs []string `json:"urls"` } -func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) { - projects, err := a.listProjects(filters) +// FetchCharts fetches charts +func (a *Adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) { + projects, err := a.ListProjects(filters) if err != nil { return nil, err } resources := []*model.Resource{} for _, project := range projects { - url := fmt.Sprintf("%s/api/chartrepo/%s/charts", a.getURL(), project.Name) + url := fmt.Sprintf("%s/api/chartrepo/%s/charts", a.url, project.Name) repositories := []*model.Repository{} - if err := a.client.Get(url, &repositories); err != nil { + if err := a.httpClient.Get(url, &repositories); err != nil { return nil, err } if len(repositories) == 0 { @@ -72,9 +72,9 @@ func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error for _, repository := range repositories { name := strings.SplitN(repository.Name, "/", 2)[1] - url := fmt.Sprintf("%s/api/chartrepo/%s/charts/%s", a.getURL(), project.Name, name) + url := fmt.Sprintf("%s/api/chartrepo/%s/charts/%s", a.url, project.Name, name) versions := []*chartVersion{} - if err := a.client.Get(url, &versions); err != nil { + if err := a.httpClient.Get(url, &versions); err != nil { return nil, err } if len(versions) == 0 { @@ -102,7 +102,7 @@ func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error for _, artifact := range artifacts { resources = append(resources, &model.Resource{ Type: model.ResourceTypeChart, - Registry: a.registry, + Registry: a.Registry, Metadata: &model.ResourceMetadata{ Repository: &model.Repository{ Name: repository.Name, @@ -117,7 +117,8 @@ func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error return resources, nil } -func (a *adapter) ChartExist(name, version string) (bool, error) { +// ChartExist checks the existence of the chart +func (a *Adapter) ChartExist(name, version string) (bool, error) { _, err := a.getChartInfo(name, version) if err == nil { return true, nil @@ -128,20 +129,21 @@ func (a *adapter) ChartExist(name, version string) (bool, error) { return false, err } -func (a *adapter) getChartInfo(name, version string) (*chartVersionDetail, error) { +func (a *Adapter) getChartInfo(name, version string) (*chartVersionDetail, error) { project, name, err := parseChartName(name) if err != nil { return nil, err } url := fmt.Sprintf("%s/api/chartrepo/%s/charts/%s/%s", a.url, project, name, version) info := &chartVersionDetail{} - if err = a.client.Get(url, info); err != nil { + if err = a.httpClient.Get(url, info); err != nil { return nil, err } return info, nil } -func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) { +// DownloadChart downloads the specific chart +func (a *Adapter) DownloadChart(name, version string) (io.ReadCloser, error) { info, err := a.getChartInfo(name, version) if err != nil { return nil, err @@ -162,7 +164,7 @@ func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) { if err != nil { return nil, err } - resp, err := a.client.Do(req) + resp, err := a.httpClient.Do(req) if err != nil { return nil, err } @@ -176,7 +178,8 @@ func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) { return resp.Body, nil } -func (a *adapter) UploadChart(name, version string, chart io.Reader) error { +// UploadChart uploads the chart +func (a *Adapter) UploadChart(name, version string, chart io.Reader) error { project, name, err := parseChartName(name) if err != nil { return err @@ -200,7 +203,7 @@ func (a *adapter) UploadChart(name, version string, chart io.Reader) error { return err } req.Header.Set("Content-Type", w.FormDataContentType()) - resp, err := a.client.Do(req) + resp, err := a.httpClient.Do(req) if err != nil { return err } @@ -219,13 +222,14 @@ func (a *adapter) UploadChart(name, version string, chart io.Reader) error { return nil } -func (a *adapter) DeleteChart(name, version string) error { +// DeleteChart deletes the chart +func (a *Adapter) DeleteChart(name, version string) error { project, name, err := parseChartName(name) if err != nil { return err } url := fmt.Sprintf("%s/api/chartrepo/%s/charts/%s/%s", a.url, project, name, version) - return a.client.Delete(url) + return a.httpClient.Delete(url) } // TODO merge this method and utils.ParseRepository? diff --git a/src/replication/adapter/harbor/chart_registry_test.go b/src/replication/adapter/harbor/base/chart_registry_test.go similarity index 93% rename from src/replication/adapter/harbor/chart_registry_test.go rename to src/replication/adapter/harbor/base/chart_registry_test.go index 88ae6edc2..f38bfc93a 100644 --- a/src/replication/adapter/harbor/chart_registry_test.go +++ b/src/replication/adapter/harbor/base/chart_registry_test.go @@ -12,15 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package harbor +package base import ( "bytes" - "fmt" "net/http" "testing" - "github.com/goharbor/harbor/src/common/api" "github.com/goharbor/harbor/src/common/utils/test" "github.com/goharbor/harbor/src/replication/model" "github.com/stretchr/testify/assert" @@ -31,7 +29,7 @@ func TestFetchCharts(t *testing.T) { server := test.NewServer([]*test.RequestHandlerMapping{ { Method: http.MethodGet, - Pattern: fmt.Sprintf("/api/%s/projects", api.APIVersion), + Pattern: "/api/projects", Handler: func(w http.ResponseWriter, r *http.Request) { data := `[{ "name": "library", @@ -69,7 +67,7 @@ func TestFetchCharts(t *testing.T) { registry := &model.Registry{ URL: server.URL, } - adapter, err := newAdapter(registry) + adapter, err := New(registry) require.Nil(t, err) // nil filter resources, err := adapter.FetchCharts(nil) @@ -116,7 +114,7 @@ func TestChartExist(t *testing.T) { registry := &model.Registry{ URL: server.URL, } - adapter, err := newAdapter(registry) + adapter, err := New(registry) require.Nil(t, err) exist, err := adapter.ChartExist("library/harbor", "1.0") require.Nil(t, err) @@ -149,7 +147,7 @@ func TestDownloadChart(t *testing.T) { registry := &model.Registry{ URL: server.URL, } - adapter, err := newAdapter(registry) + adapter, err := New(registry) require.Nil(t, err) _, err = adapter.DownloadChart("library/harbor", "1.0") require.Nil(t, err) @@ -167,7 +165,7 @@ func TestUploadChart(t *testing.T) { registry := &model.Registry{ URL: server.URL, } - adapter, err := newAdapter(registry) + adapter, err := New(registry) require.Nil(t, err) err = adapter.UploadChart("library/harbor", "1.0", bytes.NewBuffer(nil)) require.Nil(t, err) @@ -185,7 +183,7 @@ func TestDeleteChart(t *testing.T) { registry := &model.Registry{ URL: server.URL, } - adapter, err := newAdapter(registry) + adapter, err := New(registry) require.Nil(t, err) err = adapter.DeleteChart("library/harbor", "1.0") require.Nil(t, err) diff --git a/src/replication/adapter/harbor/base/client.go b/src/replication/adapter/harbor/base/client.go new file mode 100644 index 000000000..4f740ed00 --- /dev/null +++ b/src/replication/adapter/harbor/base/client.go @@ -0,0 +1,135 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package base + +import ( + "fmt" + "net/http" + "strings" + + common_http "github.com/goharbor/harbor/src/common/http" +) + +// NewClient returns an instance of the base client +func NewClient(url string, c *common_http.Client) (*Client, error) { + client := &Client{ + URL: strings.TrimSuffix(url, "/"), + C: c, + } + version, err := client.GetAPIVersion() + if err != nil { + return nil, err + } + client.APIVersion = version + return client, nil +} + +// Client is the base client that provides common methods for all versions of Harbor clients +type Client struct { + URL string + APIVersion string + C *common_http.Client +} + +// GetAPIVersion returns the supported API version +func (c *Client) GetAPIVersion() (string, error) { + version := &struct { + Version string `json:"version"` + }{} + err := c.C.Get(c.URL+"/api/version", version) + if err == nil { + return version.Version, nil + } + // Harbor 1.x has no API version endpoint + if e, ok := err.(*common_http.Error); ok && e.Code == http.StatusNotFound { + return "", nil + } + return "", err +} + +// ChartRegistryEnabled returns whether the chart registry is enabled for the Harbor instance +func (c *Client) ChartRegistryEnabled() (bool, error) { + sys := &struct { + ChartRegistryEnabled bool `json:"with_chartmuseum"` + }{} + if err := c.C.Get(c.BaseURL()+"/systeminfo", sys); err != nil { + return false, err + } + return sys.ChartRegistryEnabled, nil +} + +// ListLabels lists system level labels +func (c *Client) ListLabels() ([]string, error) { + labels := []*struct { + Name string `json:"name"` + }{} + err := c.C.Get(c.BaseURL()+"/labels?scope=g", &labels) + if err == nil { + var lbs []string + for _, label := range labels { + lbs = append(lbs, label.Name) + } + return lbs, nil + } + // label isn't supported in some previous version of Harbor + if e, ok := err.(*common_http.Error); !ok || e.Code != http.StatusNotFound { + return nil, err + } + return nil, nil +} + +// CreateProject creates project +func (c *Client) CreateProject(name string, metadata map[string]interface{}) error { + project := struct { + Name string `json:"project_name"` + Metadata map[string]interface{} `json:"metadata"` + }{ + Name: name, + Metadata: metadata, + } + return c.C.Post(c.BaseURL()+"/projects", project) +} + +// ListProjects lists projects +func (c *Client) ListProjects(name string) ([]*Project, error) { + projects := []*Project{} + url := fmt.Sprintf("%s/projects?name=%s", c.BaseURL(), name) + if err := c.C.GetAndIteratePagination(url, &projects); err != nil { + return nil, err + } + return projects, nil +} + +// GetProject gets the specific project +func (c *Client) GetProject(name string) (*Project, error) { + projects, err := c.ListProjects(name) + if err != nil { + return nil, err + } + for _, project := range projects { + if project.Name == name { + return project, nil + } + } + return nil, nil +} + +// BaseURL returns the base URL of APIs +func (c *Client) BaseURL() string { + if len(c.APIVersion) == 0 { + return fmt.Sprintf("%s/api", c.URL) + } + return fmt.Sprintf("%s/api/%s", c.URL, c.APIVersion) +} diff --git a/src/replication/adapter/harbor/image_registry.go b/src/replication/adapter/harbor/image_registry.go deleted file mode 100644 index 5f52c3e15..000000000 --- a/src/replication/adapter/harbor/image_registry.go +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package harbor - -import ( - "fmt" - "github.com/goharbor/harbor/src/common/api" - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/utils" - "github.com/goharbor/harbor/src/controller/artifact" - "github.com/goharbor/harbor/src/lib/log" - adp "github.com/goharbor/harbor/src/replication/adapter" - "github.com/goharbor/harbor/src/replication/filter" - "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/util" - "strings" -) - -func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, error) { - projects, err := a.listProjects(filters) - if err != nil { - return nil, err - } - - var resources []*model.Resource - for _, project := range projects { - repositories, err := a.listRepositories(project, filters) - if err != nil { - return nil, err - } - if len(repositories) == 0 { - continue - } - - var rawResources = make([]*model.Resource, len(repositories)) - runner := utils.NewLimitedConcurrentRunner(adp.MaxConcurrency) - defer runner.Cancel() - - for i, r := range repositories { - index := i - repo := r - runner.AddTask(func() error { - artifacts, err := a.listArtifacts(repo.Name, filters) - if err != nil { - return fmt.Errorf("failed to list artifacts of repository '%s': %v", repo.Name, err) - } - if len(artifacts) == 0 { - rawResources[index] = nil - return nil - } - - rawResources[index] = &model.Resource{ - Type: model.ResourceTypeArtifact, - Registry: a.registry, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: repo.Name, - Metadata: project.Metadata, - }, - Artifacts: artifacts, - }, - } - return nil - }) - } - runner.Wait() - - if runner.IsCancelled() { - return nil, fmt.Errorf("FetchArtifacts error when collect tags for repos") - } - - for _, r := range rawResources { - if r != nil { - resources = append(resources, r) - } - } - } - - return resources, nil -} - -func (a *adapter) listProjects(filters []*model.Filter) ([]*project, error) { - pattern := "" - for _, filter := range filters { - if filter.Type == model.FilterTypeName { - pattern = filter.Value.(string) - break - } - } - var projects []*project - if len(pattern) > 0 { - substrings := strings.Split(pattern, "/") - projectPattern := substrings[0] - names, ok := util.IsSpecificPathComponent(projectPattern) - if ok { - for _, name := range names { - project, err := a.getProject(name) - if err != nil { - return nil, err - } - if project == nil { - continue - } - projects = append(projects, project) - } - } - } - if len(projects) > 0 { - var names []string - for _, project := range projects { - names = append(names, project.Name) - } - log.Debugf("parsed the projects %v from pattern %s", names, pattern) - return projects, nil - } - return a.getProjects("") -} - -func (a *adapter) listRepositories(project *project, filters []*model.Filter) ([]*model.Repository, error) { - repositories := []*models.RepoRecord{} - url := fmt.Sprintf("%s/api/%s/projects/%s/repositories", a.getURL(), api.APIVersion, project.Name) - if err := a.client.GetAndIteratePagination(url, &repositories); err != nil { - return nil, err - } - var repos []*model.Repository - for _, repository := range repositories { - repos = append(repos, &model.Repository{ - Name: repository.Name, - Metadata: project.Metadata, - }) - } - return filter.DoFilterRepositories(repos, filters) -} - -func (a *adapter) listArtifacts(repository string, filters []*model.Filter) ([]*model.Artifact, error) { - project, repository := utils.ParseRepository(repository) - url := fmt.Sprintf("%s/api/%s/projects/%s/repositories/%s/artifacts?with_label=true", - a.getURL(), api.APIVersion, project, repository) - artifacts := []*artifact.Artifact{} - if err := a.client.GetAndIteratePagination(url, &artifacts); err != nil { - return nil, err - } - var arts []*model.Artifact - for _, artifact := range artifacts { - art := &model.Artifact{ - Type: artifact.Type, - Digest: artifact.Digest, - } - for _, label := range artifact.Labels { - art.Labels = append(art.Labels, label.Name) - } - for _, tag := range artifact.Tags { - art.Tags = append(art.Tags, tag.Name) - } - arts = append(arts, art) - } - return filter.DoFilterArtifacts(arts, filters) -} - -func (a *adapter) DeleteTag(repository, tag string) error { - project, repository := utils.ParseRepository(repository) - url := fmt.Sprintf("%s/api/%s/projects/%s/repositories/%s/artifacts/%s/tags/%s", - a.getURL(), api.APIVersion, project, repository, tag, tag) - return a.client.Delete(url) -} diff --git a/src/replication/adapter/harbor/image_registry_test.go b/src/replication/adapter/harbor/image_registry_test.go deleted file mode 100644 index 9153303ca..000000000 --- a/src/replication/adapter/harbor/image_registry_test.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package harbor - -import ( - "net/http" - "testing" - - "github.com/goharbor/harbor/src/common/utils/test" - "github.com/goharbor/harbor/src/replication/model" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestFetchArtifacts(t *testing.T) { - server := test.NewServer([]*test.RequestHandlerMapping{ - { - Method: http.MethodGet, - Pattern: "/api/v2.0/projects/library/repositories/hello-world/artifacts", - Handler: func(w http.ResponseWriter, r *http.Request) { - data := `[ - { - "digest": "digest1", - "tags": [ - { - "name": "1.0" - } - ] - }, - { - "digest": "digest2", - "tags": [ - { - "name": "2.0" - } - ] - } -]` - w.Write([]byte(data)) - }, - }, - { - Method: http.MethodGet, - Pattern: "/api/v2.0/projects/library/repositories", - Handler: func(w http.ResponseWriter, r *http.Request) { - data := `[{ - "name": "library/hello-world" - }]` - w.Write([]byte(data)) - }, - }, - { - Method: http.MethodGet, - Pattern: "/api/v2.0/projects", - Handler: func(w http.ResponseWriter, r *http.Request) { - data := `[{ - "name": "library", - "metadata": {"public":true} - }]` - w.Write([]byte(data)) - }, - }, - }...) - defer server.Close() - registry := &model.Registry{ - URL: server.URL, - } - adapter, err := newAdapter(registry) - require.Nil(t, err) - // nil filter - resources, err := adapter.FetchArtifacts(nil) - require.Nil(t, err) - assert.Equal(t, 1, len(resources)) - assert.Equal(t, model.ResourceTypeArtifact, resources[0].Type) - assert.Equal(t, "library/hello-world", resources[0].Metadata.Repository.Name) - assert.Equal(t, 2, len(resources[0].Metadata.Artifacts)) - assert.Equal(t, "1.0", resources[0].Metadata.Artifacts[0].Tags[0]) - assert.Equal(t, "2.0", resources[0].Metadata.Artifacts[1].Tags[0]) - // not nil filter - filters := []*model.Filter{ - { - Type: model.FilterTypeName, - Value: "library/*", - }, - { - Type: model.FilterTypeTag, - Value: "1.0", - }, - } - resources, err = adapter.FetchArtifacts(filters) - require.Nil(t, err) - assert.Equal(t, 1, len(resources)) - assert.Equal(t, model.ResourceTypeArtifact, resources[0].Type) - assert.Equal(t, "library/hello-world", resources[0].Metadata.Repository.Name) - assert.Equal(t, 1, len(resources[0].Metadata.Artifacts)) - assert.Equal(t, "1.0", resources[0].Metadata.Artifacts[0].Tags[0]) -} diff --git a/src/replication/adapter/harbor/v1/adapter.go b/src/replication/adapter/harbor/v1/adapter.go new file mode 100644 index 000000000..a10d5e9b4 --- /dev/null +++ b/src/replication/adapter/harbor/v1/adapter.go @@ -0,0 +1,126 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "fmt" + + "github.com/goharbor/harbor/src/common/utils" + adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/adapter/harbor/base" + "github.com/goharbor/harbor/src/replication/filter" + "github.com/goharbor/harbor/src/replication/model" +) + +var _ adp.Adapter = &adapter{} +var _ adp.ArtifactRegistry = &adapter{} +var _ adp.ChartRegistry = &adapter{} + +// New creates a Adapter for Harbor 1.x +func New(base *base.Adapter) adp.Adapter { + return &adapter{ + Adapter: base, + client: &client{Client: base.Client}, + } +} + +type adapter struct { + *base.Adapter + client *client +} + +func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, error) { + projects, err := a.ListProjects(filters) + if err != nil { + return nil, err + } + + var resources []*model.Resource + for _, project := range projects { + repositories, err := a.listRepositories(project, filters) + if err != nil { + return nil, err + } + if len(repositories) == 0 { + continue + } + + var rawResources = make([]*model.Resource, len(repositories)) + runner := utils.NewLimitedConcurrentRunner(adp.MaxConcurrency) + defer runner.Cancel() + + for i, r := range repositories { + index := i + repo := r + runner.AddTask(func() error { + artifacts, err := a.listArtifacts(repo.Name, filters) + if err != nil { + return fmt.Errorf("failed to list artifacts of repository '%s': %v", repo.Name, err) + } + if len(artifacts) == 0 { + rawResources[index] = nil + return nil + } + + rawResources[index] = &model.Resource{ + Type: model.ResourceTypeImage, + Registry: a.Registry, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: repo.Name, + Metadata: project.Metadata, + }, + Artifacts: artifacts, + }, + } + return nil + }) + } + runner.Wait() + + if runner.IsCancelled() { + return nil, fmt.Errorf("FetchArtifacts error when collect tags for repos") + } + + for _, r := range rawResources { + if r != nil { + resources = append(resources, r) + } + } + } + + return resources, nil +} + +// override the default implementation by calling Harbor API directly +func (a *adapter) DeleteManifest(repository, reference string) error { + return a.client.deleteManifest(repository, reference) +} + +func (a *adapter) listRepositories(project *base.Project, filters []*model.Filter) ([]*model.Repository, error) { + repositories, err := a.client.listRepositories(project) + if err != nil { + return nil, err + } + return filter.DoFilterRepositories(repositories, filters) +} + +func (a *adapter) listArtifacts(repository string, filters []*model.Filter) ([]*model.Artifact, error) { + artifacts, err := a.client.listArtifacts(repository) + if err != nil { + return nil, err + } + return filter.DoFilterArtifacts(artifacts, filters) +} diff --git a/src/replication/adapter/harbor/v1/client.go b/src/replication/adapter/harbor/v1/client.go new file mode 100644 index 000000000..763347967 --- /dev/null +++ b/src/replication/adapter/harbor/v1/client.go @@ -0,0 +1,73 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "fmt" + + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/replication/adapter/harbor/base" + "github.com/goharbor/harbor/src/replication/model" +) + +type client struct { + *base.Client +} + +func (c *client) listRepositories(project *base.Project) ([]*model.Repository, error) { + repositories := []*models.RepoRecord{} + url := fmt.Sprintf("%s/repositories?project_id=%d", c.BaseURL(), project.ID) + if err := c.C.GetAndIteratePagination(url, &repositories); err != nil { + return nil, err + } + var repos []*model.Repository + for _, repository := range repositories { + repos = append(repos, &model.Repository{ + Name: repository.Name, + Metadata: project.Metadata, + }) + } + return repos, nil +} + +func (c *client) listArtifacts(repository string) ([]*model.Artifact, error) { + url := fmt.Sprintf("%s/repositories/%s/tags", c.BaseURL(), repository) + tags := []*struct { + Name string `json:"name"` + Labels []*struct { + Name string `json:"name"` + } + }{} + if err := c.C.Get(url, &tags); err != nil { + return nil, err + } + var artifacts []*model.Artifact + for _, tag := range tags { + artifact := &model.Artifact{ + Type: string(model.ResourceTypeImage), + Tags: []string{tag.Name}, + } + for _, label := range tag.Labels { + artifact.Labels = append(artifact.Labels, label.Name) + } + artifacts = append(artifacts, artifact) + } + return artifacts, nil +} + +func (c *client) deleteManifest(repository, reference string) error { + url := fmt.Sprintf("%s/repositories/%s/tags/%s", c.BaseURL(), repository, reference) + return c.C.Delete(url) +} diff --git a/src/replication/adapter/harbor/v2/adapter.go b/src/replication/adapter/harbor/v2/adapter.go new file mode 100644 index 000000000..8edd96c0b --- /dev/null +++ b/src/replication/adapter/harbor/v2/adapter.go @@ -0,0 +1,134 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v2 + +import ( + "fmt" + + "github.com/goharbor/harbor/src/common/utils" + adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/adapter/harbor/base" + "github.com/goharbor/harbor/src/replication/filter" + "github.com/goharbor/harbor/src/replication/model" +) + +var _ adp.Adapter = &adapter{} +var _ adp.ArtifactRegistry = &adapter{} +var _ adp.ChartRegistry = &adapter{} + +// New creates a Adapter for Harbor 2.x +func New(base *base.Adapter) adp.Adapter { + return &adapter{ + Adapter: base, + client: &client{Client: base.Client}, + } +} + +type adapter struct { + *base.Adapter + client *client +} + +func (a *adapter) Info() (*model.RegistryInfo, error) { + info, err := a.Adapter.Info() + if err != nil { + return nil, err + } + info.SupportedResourceTypes = append(info.SupportedResourceTypes, model.ResourceTypeArtifact) + return info, err +} + +func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, error) { + projects, err := a.ListProjects(filters) + if err != nil { + return nil, err + } + + var resources []*model.Resource + for _, project := range projects { + repositories, err := a.listRepositories(project, filters) + if err != nil { + return nil, err + } + if len(repositories) == 0 { + continue + } + + var rawResources = make([]*model.Resource, len(repositories)) + runner := utils.NewLimitedConcurrentRunner(adp.MaxConcurrency) + defer runner.Cancel() + + for i, r := range repositories { + index := i + repo := r + runner.AddTask(func() error { + artifacts, err := a.listArtifacts(repo.Name, filters) + if err != nil { + return fmt.Errorf("failed to list artifacts of repository '%s': %v", repo.Name, err) + } + if len(artifacts) == 0 { + rawResources[index] = nil + return nil + } + + rawResources[index] = &model.Resource{ + Type: model.ResourceTypeArtifact, + Registry: a.Registry, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: repo.Name, + Metadata: project.Metadata, + }, + Artifacts: artifacts, + }, + } + return nil + }) + } + runner.Wait() + + if runner.IsCancelled() { + return nil, fmt.Errorf("FetchArtifacts error when collect tags for repos") + } + + for _, r := range rawResources { + if r != nil { + resources = append(resources, r) + } + } + } + + return resources, nil +} + +func (a *adapter) DeleteTag(repository, tag string) error { + return a.client.deleteTag(repository, tag) +} + +func (a *adapter) listRepositories(project *base.Project, filters []*model.Filter) ([]*model.Repository, error) { + repositories, err := a.client.listRepositories(project) + if err != nil { + return nil, err + } + return filter.DoFilterRepositories(repositories, filters) +} + +func (a *adapter) listArtifacts(repository string, filters []*model.Filter) ([]*model.Artifact, error) { + artifacts, err := a.client.listArtifacts(repository) + if err != nil { + return nil, err + } + return filter.DoFilterArtifacts(artifacts, filters) +} diff --git a/src/replication/adapter/harbor/v2/client.go b/src/replication/adapter/harbor/v2/client.go new file mode 100644 index 000000000..f9a585ca3 --- /dev/null +++ b/src/replication/adapter/harbor/v2/client.go @@ -0,0 +1,80 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v2 + +import ( + "fmt" + "net/url" + + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/controller/artifact" + "github.com/goharbor/harbor/src/replication/adapter/harbor/base" + "github.com/goharbor/harbor/src/replication/model" +) + +type client struct { + *base.Client +} + +func (c *client) listRepositories(project *base.Project) ([]*model.Repository, error) { + repositories := []*models.RepoRecord{} + url := fmt.Sprintf("%s/projects/%s/repositories", c.BaseURL(), project.Name) + if err := c.C.GetAndIteratePagination(url, &repositories); err != nil { + return nil, err + } + var repos []*model.Repository + for _, repository := range repositories { + repos = append(repos, &model.Repository{ + Name: repository.Name, + Metadata: project.Metadata, + }) + } + return repos, nil +} + +func (c *client) listArtifacts(repository string) ([]*model.Artifact, error) { + project, repository := utils.ParseRepository(repository) + repository = url.PathEscape(url.PathEscape(repository)) + url := fmt.Sprintf("%s/projects/%s/repositories/%s/artifacts?with_label=true", + c.BaseURL(), project, repository) + artifacts := []*artifact.Artifact{} + if err := c.C.GetAndIteratePagination(url, &artifacts); err != nil { + return nil, err + } + var arts []*model.Artifact + for _, artifact := range artifacts { + art := &model.Artifact{ + Type: artifact.Type, + Digest: artifact.Digest, + } + for _, label := range artifact.Labels { + art.Labels = append(art.Labels, label.Name) + } + for _, tag := range artifact.Tags { + art.Tags = append(art.Tags, tag.Name) + } + arts = append(arts, art) + } + return arts, nil +} + +func (c *client) deleteTag(repository, tag string) error { + project, repository := utils.ParseRepository(repository) + repository = url.PathEscape(url.PathEscape(repository)) + url := fmt.Sprintf("%s/projects/%s/repositories/%s/artifacts/%s/tags/%s", + c.BaseURL(), project, repository, tag, tag) + return c.C.Delete(url) +} diff --git a/src/server/route.go b/src/server/route.go index 4a5f65952..11ee6c357 100644 --- a/src/server/route.go +++ b/src/server/route.go @@ -24,9 +24,14 @@ import ( "github.com/goharbor/harbor/src/core/service/notifications/jobs" "github.com/goharbor/harbor/src/core/service/notifications/scheduler" "github.com/goharbor/harbor/src/core/service/token" + "github.com/goharbor/harbor/src/server/router" + "net/http" ) func registerRoutes() { + // API version + router.NewRoute().Method(http.MethodGet).Path("/api/version").HandlerFunc(GetAPIVersion) + // Controller API: beego.Router("/c/login", &controllers.CommonController{}, "post:Login") beego.Router("/c/log_out", &controllers.CommonController{}, "get:LogOut") diff --git a/src/server/v2.0/route/legacy.go b/src/server/v2.0/route/legacy.go index d4da50763..5891da464 100755 --- a/src/server/v2.0/route/legacy.go +++ b/src/server/v2.0/route/legacy.go @@ -23,6 +23,7 @@ import ( // RegisterRoutes for Harbor legacy APIs // TODO bump up the version of APIs called by clients func registerLegacyRoutes() { + version := APIVersion beego.Router("/api/"+version+"/projects/:pid([0-9]+)/members/?:pmid([0-9]+)", &api.ProjectMemberAPI{}) beego.Router("/api/"+version+"/projects/", &api.ProjectAPI{}, "head:Head") beego.Router("/api/"+version+"/projects/:id([0-9]+)", &api.ProjectAPI{}) diff --git a/src/server/v2.0/route/route.go b/src/server/v2.0/route/route.go index bf7bd8fca..b9e0df2a9 100644 --- a/src/server/v2.0/route/route.go +++ b/src/server/v2.0/route/route.go @@ -20,14 +20,15 @@ import ( "github.com/goharbor/harbor/src/server/v2.0/handler" ) +// const definition const ( - version = "v2.0" + APIVersion = "v2.0" ) // RegisterRoutes for Harbor v2.0 APIs func RegisterRoutes() { registerLegacyRoutes() - router.NewRoute().Path("/api/" + version + "/*"). - Middleware(apiversion.Middleware(version)). + router.NewRoute().Path("/api/" + APIVersion + "/*"). + Middleware(apiversion.Middleware(APIVersion)). Handler(handler.New()) } diff --git a/src/server/version.go b/src/server/version.go new file mode 100644 index 000000000..b13baec51 --- /dev/null +++ b/src/server/version.go @@ -0,0 +1,39 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "encoding/json" + "net/http" + + serror "github.com/goharbor/harbor/src/server/error" + "github.com/goharbor/harbor/src/server/v2.0/route" +) + +var ( + version = route.APIVersion +) + +// APIVersion model +type APIVersion struct { + Version string `json:"version"` +} + +// GetAPIVersion returns the current supported API version +func GetAPIVersion(w http.ResponseWriter, r *http.Request) { + if err := json.NewEncoder(w).Encode(&APIVersion{Version: version}); err != nil { + serror.SendError(w, err) + } +}