From 28714f8b70e0e03983dea0ab5663adb60113d342 Mon Sep 17 00:00:00 2001 From: peimingming Date: Wed, 6 Jan 2021 14:47:38 +0800 Subject: [PATCH] Support artifact hub replication using new API Signed-off-by: peimingming --- src/replication/adapter/adapter.go | 2 +- .../adapter/artifacthub/adapter_test.go | 6 +- .../adapter/artifacthub/chart_registry.go | 104 ++++++++------ src/replication/adapter/artifacthub/client.go | 132 +++++------------- src/replication/adapter/artifacthub/consts.go | 3 +- src/replication/adapter/artifacthub/model.go | 16 ++- .../adapter/harbor/base/chart_registry.go | 2 +- .../harbor/base/chart_registry_test.go | 2 +- .../adapter/helmhub/chart_registry.go | 2 +- .../adapter/tencentcr/chart_registry.go | 2 +- src/replication/transfer/chart/transfer.go | 17 ++- .../transfer/chart/transfer_test.go | 2 +- 12 files changed, 139 insertions(+), 151 deletions(-) diff --git a/src/replication/adapter/adapter.go b/src/replication/adapter/adapter.go index ab7ef536d..6c5abb8dd 100644 --- a/src/replication/adapter/adapter.go +++ b/src/replication/adapter/adapter.go @@ -68,7 +68,7 @@ type ArtifactRegistry interface { type ChartRegistry interface { FetchCharts(filters []*model.Filter) ([]*model.Resource, error) ChartExist(name, version string) (bool, error) - DownloadChart(name, version string) (io.ReadCloser, error) + DownloadChart(name, version, contentURL string) (io.ReadCloser, error) UploadChart(name, version string, chart io.Reader) error DeleteChart(name, version string) error } diff --git a/src/replication/adapter/artifacthub/adapter_test.go b/src/replication/adapter/artifacthub/adapter_test.go index 0eebdfeb0..e50fc2147 100644 --- a/src/replication/adapter/artifacthub/adapter_test.go +++ b/src/replication/adapter/artifacthub/adapter_test.go @@ -82,7 +82,11 @@ func TestAdapter_DownloadChart(t *testing.T) { URL: "https://artifacthub.io", }) - data, err := a.DownloadChart("harbor/harbor", "1.5.0") + data, err := a.DownloadChart("harbor/harbor", "1.5.0", "") + assert.NotNil(t, err) + assert.Nil(t, data) + + data, err = a.DownloadChart("harbor/harbor", "1.5.0", "https://helm.goharbor.io/harbor-1.5.0.tgz") assert.Nil(t, err) assert.NotNil(t, data) } diff --git a/src/replication/adapter/artifacthub/chart_registry.go b/src/replication/adapter/artifacthub/chart_registry.go index d7c9d08f2..c261423a2 100644 --- a/src/replication/adapter/artifacthub/chart_registry.go +++ b/src/replication/adapter/artifacthub/chart_registry.go @@ -17,7 +17,6 @@ package artifacthub import ( "fmt" "github.com/goharbor/harbor/src/lib/errors" - "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/replication/filter" "github.com/goharbor/harbor/src/replication/model" "io" @@ -26,62 +25,92 @@ import ( ) func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) { - pkgs, err := a.client.getAllPackages(HelmChart) + charts, err := a.client.getReplicationInfo() if err != nil { return nil, err } resources := []*model.Resource{} var repositories []*model.Repository - for _, pkg := range pkgs { - repositories = append(repositories, &model.Repository{ - Name: fmt.Sprintf("%s/%s", pkg.Repository.Name, pkg.Name), - }) + var artifacts []*model.Artifact + repoSet := map[string]interface{}{} + versionSet := map[string]interface{}{} + for _, chart := range charts { + name := fmt.Sprintf("%s/%s", chart.Repository, chart.Package) + if _, ok := repoSet[name]; !ok { + repoSet[name] = nil + repositories = append(repositories, &model.Repository{ + Name: name, + }) + } } repositories, err = filter.DoFilterRepositories(repositories, filters) if err != nil { return nil, err } + if len(repositories) == 0 { + return resources, nil + } - for _, repository := range repositories { - pkgDetail, err := a.client.getHelmPackageDetail(repository.Name) - if err != nil { - log.Errorf("fetch package detail: %v", err) - return nil, err + if len(repoSet) != len(repositories) { + repoSet = map[string]interface{}{} + for _, repo := range repositories { + repoSet[repo.Name] = nil } + } - var artifacts []*model.Artifact - for _, version := range pkgDetail.AvailableVersions { + for _, chart := range charts { + name := fmt.Sprintf("%s/%s", chart.Repository, chart.Package) + if _, ok := repoSet[name]; ok { artifacts = append(artifacts, &model.Artifact{ - Tags: []string{version.Version}, + Tags: []string{chart.Version}, }) } + } - artifacts, err = filter.DoFilterArtifacts(artifacts, filters) - if err != nil { - return nil, err - } - if len(artifacts) == 0 { + artifacts, err = filter.DoFilterArtifacts(artifacts, filters) + if err != nil { + return nil, err + } + if len(artifacts) == 0 { + return resources, nil + } + + for _, arti := range artifacts { + versionSet[arti.Tags[0]] = nil + } + + for _, chart := range charts { + name := fmt.Sprintf("%s/%s", chart.Repository, chart.Package) + if _, ok := repoSet[name]; !ok { continue } - - for _, artifact := range artifacts { - resources = append(resources, &model.Resource{ - Type: model.ResourceTypeChart, - Registry: a.registry, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: repository.Name, - }, - Artifacts: []*model.Artifact{artifact}, - }, - }) + if _, ok := versionSet[chart.Version]; !ok { + continue } + resources = append(resources, &model.Resource{ + Type: model.ResourceTypeChart, + Registry: a.registry, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: name, + }, + Artifacts: []*model.Artifact{ + { + Tags: []string{chart.Version}, + }, + }, + }, + ExtendedInfo: map[string]interface{}{ + "contentURL": chart.ContentURL, + }, + }) } return resources, nil } +// ChartExist will never be used, for this function is only used when endpoint is destination func (a *adapter) ChartExist(name, version string) (bool, error) { _, err := a.client.getHelmChartVersion(name, version) if err != nil { @@ -93,16 +122,11 @@ func (a *adapter) ChartExist(name, version string) (bool, error) { return true, nil } -func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) { - chartVersion, err := a.client.getHelmChartVersion(name, version) - if err != nil { - return nil, err +func (a *adapter) DownloadChart(name, version, contentURL string) (io.ReadCloser, error) { + if len(contentURL) == 0 { + return nil, errors.Errorf("empty chart content url, %s:%s", name, version) } - - if len(chartVersion.ContentURL) == 0 { - return nil, errors.Errorf("") - } - return a.download(chartVersion.ContentURL) + return a.download(contentURL) } func (a *adapter) download(contentURL string) (io.ReadCloser, error) { diff --git a/src/replication/adapter/artifacthub/client.go b/src/replication/adapter/artifacthub/client.go index c9f3b6013..948ed37bf 100644 --- a/src/replication/adapter/artifacthub/client.go +++ b/src/replication/adapter/artifacthub/client.go @@ -24,101 +24,6 @@ func newClient(registry *model.Registry) *Client { } } -// searchPackages query the artifact package list from artifact hub. -func (c *Client) searchPackages(kind, offset, limit int, queryString string) (*PackageResponse, error) { - request, err := http.NewRequest(http.MethodGet, baseURL+searchPackages(kind, offset, limit, queryString), nil) - if err != nil { - return nil, err - } - - resp, err := c.do(request) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - if resp.StatusCode != http.StatusOK { - msg := &Message{} - err = json.Unmarshal(body, msg) - if err != nil { - msg.Message = string(body) - } - return nil, fmt.Errorf("search package list error %d: %s", resp.StatusCode, msg.Message) - } - - packageResp := &PackageResponse{} - err = json.Unmarshal(body, packageResp) - if err != nil { - return nil, fmt.Errorf("unmarshal package list response error: %v", err) - } - return packageResp, nil -} - -// getAllPackages gets all of the specific kind of artifact packages from artifact hub. -func (c *Client) getAllPackages(kind int) (pkgs []*Package, err error) { - offset := 0 - limit := 50 - shouldContinue := true - // todo: rate limit - for shouldContinue { - pkgResp, err := c.searchPackages(HelmChart, offset, limit, "") - if err != nil { - return nil, err - } - - pkgs = append(pkgs, pkgResp.Data.Packages...) - total := pkgResp.Metadata.Total - offset = offset + limit - if offset >= total { - shouldContinue = false - } - } - return pkgs, nil -} - -// getHelmPackageDetail get the chart detail of a helm chart from artifact hub. -func (c *Client) getHelmPackageDetail(fullName string) (*PackageDetail, error) { - request, err := http.NewRequest(http.MethodGet, baseURL+getHelmPackageDetail(fullName), nil) - if err != nil { - return nil, err - } - - resp, err := c.do(request) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - if resp.StatusCode == http.StatusNotFound { - return nil, ErrHTTPNotFound - } else if resp.StatusCode != http.StatusOK { - msg := &Message{} - err = json.Unmarshal(body, msg) - if err != nil { - msg.Message = string(body) - } - return nil, fmt.Errorf("fetch package detail error %d: %s", resp.StatusCode, msg.Message) - } - - pkgDetail := &PackageDetail{} - err = json.Unmarshal(body, pkgDetail) - if err != nil { - return nil, fmt.Errorf("unmarshal package detail response error: %v", err) - } - - return pkgDetail, nil -} - // getHelmVersion get the package version of a helm chart from artifact hub. func (c *Client) getHelmChartVersion(fullName, version string) (*ChartVersion, error) { request, err := http.NewRequest(http.MethodGet, baseURL+getHelmVersion(fullName, version), nil) @@ -157,6 +62,43 @@ func (c *Client) getHelmChartVersion(fullName, version string) (*ChartVersion, e return chartVersion, nil } +// getReplicationInfo gets the brief info of all helm chart from artifact hub. +// see https://github.com/artifacthub/hub/issues/997 +func (c *Client) getReplicationInfo() ([]*ChartInfo, error) { + request, err := http.NewRequest(http.MethodGet, baseURL+getReplicationInfo, nil) + if err != nil { + return nil, err + } + + resp, err := c.do(request) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + msg := &Message{} + err = json.Unmarshal(body, msg) + if err != nil { + msg.Message = string(body) + } + return nil, fmt.Errorf("get chart replication info error %d: %s", resp.StatusCode, msg.Message) + } + + var chartInfo []*ChartInfo + err = json.Unmarshal(body, &chartInfo) + if err != nil { + return nil, fmt.Errorf("unmarshal chart replication info error: %v", err) + } + + return chartInfo, nil +} + func (c *Client) checkHealthy() error { request, err := http.NewRequest(http.MethodGet, baseURL, nil) if err != nil { diff --git a/src/replication/adapter/artifacthub/consts.go b/src/replication/adapter/artifacthub/consts.go index 3de83b1bb..51ba97d86 100644 --- a/src/replication/adapter/artifacthub/consts.go +++ b/src/replication/adapter/artifacthub/consts.go @@ -6,7 +6,8 @@ import ( ) const ( - baseURL = "https://artifacthub.io" + baseURL = "https://artifacthub.io" + getReplicationInfo = "/api/v1/harborReplication" ) const ( diff --git a/src/replication/adapter/artifacthub/model.go b/src/replication/adapter/artifacthub/model.go index 8e58029fd..d32622d1c 100644 --- a/src/replication/adapter/artifacthub/model.go +++ b/src/replication/adapter/artifacthub/model.go @@ -13,9 +13,10 @@ type PackageData struct { // Package ... type Package struct { - PackageID string `json:"package_id"` - Name string `json:"name"` - Repository *Repository `json:"repository"` + PackageID string `json:"package_id"` + Name string `json:"name"` + NormalizedName string `json:"normalized_name"` + Repository *Repository `json:"repository"` } // Repository ... @@ -29,6 +30,7 @@ type Repository struct { type PackageDetail struct { PackageID string `json:"package_id"` Name string `json:"name"` + NormalizedName string `json:"normalized_name"` Version string `json:"version"` AppVersion string `json:"app_version"` Repository RepositoryDetail `json:"repository"` @@ -70,3 +72,11 @@ type Metadata struct { type Message struct { Message string `json:"message"` } + +// ChartInfo ... +type ChartInfo struct { + Repository string `json:"repository"` + Package string `json:"package"` + Version string `json:"version"` + ContentURL string `json:"url"` +} diff --git a/src/replication/adapter/harbor/base/chart_registry.go b/src/replication/adapter/harbor/base/chart_registry.go index bcdba7894..ac13daac5 100644 --- a/src/replication/adapter/harbor/base/chart_registry.go +++ b/src/replication/adapter/harbor/base/chart_registry.go @@ -144,7 +144,7 @@ func (a *Adapter) getChartInfo(name, version string) (*chartVersionDetail, error } // DownloadChart downloads the specific chart -func (a *Adapter) DownloadChart(name, version string) (io.ReadCloser, error) { +func (a *Adapter) DownloadChart(name, version, contentURL string) (io.ReadCloser, error) { info, err := a.getChartInfo(name, version) if err != nil { return nil, err diff --git a/src/replication/adapter/harbor/base/chart_registry_test.go b/src/replication/adapter/harbor/base/chart_registry_test.go index f38bfc93a..456f3122b 100644 --- a/src/replication/adapter/harbor/base/chart_registry_test.go +++ b/src/replication/adapter/harbor/base/chart_registry_test.go @@ -149,7 +149,7 @@ func TestDownloadChart(t *testing.T) { } adapter, err := New(registry) require.Nil(t, err) - _, err = adapter.DownloadChart("library/harbor", "1.0") + _, err = adapter.DownloadChart("library/harbor", "1.0", "") require.Nil(t, err) } diff --git a/src/replication/adapter/helmhub/chart_registry.go b/src/replication/adapter/helmhub/chart_registry.go index 9f0a9b08b..efbe029a1 100644 --- a/src/replication/adapter/helmhub/chart_registry.go +++ b/src/replication/adapter/helmhub/chart_registry.go @@ -101,7 +101,7 @@ func (a *adapter) ChartExist(name, version string) (bool, error) { return false, nil } -func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) { +func (a *adapter) DownloadChart(name, version, contentURL string) (io.ReadCloser, error) { versionList, err := a.client.fetchChartDetail(name) if err != nil { return nil, err diff --git a/src/replication/adapter/tencentcr/chart_registry.go b/src/replication/adapter/tencentcr/chart_registry.go index 6ed7b19bd..609ab7fa0 100644 --- a/src/replication/adapter/tencentcr/chart_registry.go +++ b/src/replication/adapter/tencentcr/chart_registry.go @@ -146,7 +146,7 @@ func (a *adapter) getChartInfo(name, version string) (info *tcrChartVersionDetai return } -func (a *adapter) DownloadChart(name, version string) (rc io.ReadCloser, err error) { +func (a *adapter) DownloadChart(name, version, contentURL string) (rc io.ReadCloser, err error) { var info *tcrChartVersionDetail info, err = a.getChartInfo(name, version) if err != nil { diff --git a/src/replication/transfer/chart/transfer.go b/src/replication/transfer/chart/transfer.go index 20fd846b7..b4c20ae56 100644 --- a/src/replication/transfer/chart/transfer.go +++ b/src/replication/transfer/chart/transfer.go @@ -37,8 +37,9 @@ func factory(logger trans.Logger, stopFunc trans.StopFunc) (trans.Transfer, erro } type chart struct { - name string - version string + name string + version string + contentURL string } type transfer struct { @@ -62,9 +63,15 @@ func (t *transfer) Transfer(src *model.Resource, dst *model.Resource) error { }) } + var contentURL string + if len(src.ExtendedInfo) > 0 { + contentURL = src.ExtendedInfo["contentURL"].(string) + } + srcChart := &chart{ - name: src.Metadata.Repository.Name, - version: src.Metadata.Artifacts[0].Tags[0], + name: src.Metadata.Repository.Name, + version: src.Metadata.Artifacts[0].Tags[0], + contentURL: contentURL, } dstChart := &chart{ name: dst.Metadata.Repository.Name, @@ -151,7 +158,7 @@ func (t *transfer) copy(src, dst *chart, override bool) error { } // copy the chart between the source and destination registries - chart, err := t.src.DownloadChart(src.name, src.version) + chart, err := t.src.DownloadChart(src.name, src.version, src.contentURL) if err != nil { t.logger.Errorf("failed to download the chart %s:%s: %v", src.name, src.version, err) return err diff --git a/src/replication/transfer/chart/transfer_test.go b/src/replication/transfer/chart/transfer_test.go index aa6430555..390934daa 100644 --- a/src/replication/transfer/chart/transfer_test.go +++ b/src/replication/transfer/chart/transfer_test.go @@ -45,7 +45,7 @@ func (f *fakeRegistry) FetchCharts(filters []*model.Filter) ([]*model.Resource, func (f *fakeRegistry) ChartExist(name, version string) (bool, error) { return true, nil } -func (f *fakeRegistry) DownloadChart(name, version string) (io.ReadCloser, error) { +func (f *fakeRegistry) DownloadChart(name, version, contentURL string) (io.ReadCloser, error) { r := ioutil.NopCloser(bytes.NewReader([]byte{'a'})) return r, nil }