Support artifact hub replication using new API

Signed-off-by: peimingming <peimingming@corp.netease.com>

src/replication/adapter/adapter.go

@@ -68,7 +68,7 @@ type ArtifactRegistry interface {
 type ChartRegistry interface {
 	FetchCharts(filters []*model.Filter) ([]*model.Resource, error)
 	ChartExist(name, version string) (bool, error)
-	DownloadChart(name, version string) (io.ReadCloser, error)
+	DownloadChart(name, version, contentURL string) (io.ReadCloser, error)
 	UploadChart(name, version string, chart io.Reader) error
 	DeleteChart(name, version string) error
 }

src/replication/adapter/artifacthub/adapter_test.go

@@ -82,7 +82,11 @@ func TestAdapter_DownloadChart(t *testing.T) {
 		URL:  "https://artifacthub.io",
 	})
 
-	data, err := a.DownloadChart("harbor/harbor", "1.5.0")
+	data, err := a.DownloadChart("harbor/harbor", "1.5.0", "")
+	assert.NotNil(t, err)
+	assert.Nil(t, data)
+
+	data, err = a.DownloadChart("harbor/harbor", "1.5.0", "https://helm.goharbor.io/harbor-1.5.0.tgz")
 	assert.Nil(t, err)
 	assert.NotNil(t, data)
 }

src/replication/adapter/artifacthub/chart_registry.go

@@ -17,7 +17,6 @@ package artifacthub
 import (
 	"fmt"
 	"github.com/goharbor/harbor/src/lib/errors"
-	"github.com/goharbor/harbor/src/lib/log"
 	"github.com/goharbor/harbor/src/replication/filter"
 	"github.com/goharbor/harbor/src/replication/model"
 	"io"
@@ -26,62 +25,92 @@ import (
 )
 
 func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) {
-	pkgs, err := a.client.getAllPackages(HelmChart)
+	charts, err := a.client.getReplicationInfo()
 	if err != nil {
 		return nil, err
 	}
 
 	resources := []*model.Resource{}
 	var repositories []*model.Repository
-	for _, pkg := range pkgs {
+	var artifacts []*model.Artifact
-		repositories = append(repositories, &model.Repository{
+	repoSet := map[string]interface{}{}
-			Name: fmt.Sprintf("%s/%s", pkg.Repository.Name, pkg.Name),
+	versionSet := map[string]interface{}{}
-		})
+	for _, chart := range charts {
+		name := fmt.Sprintf("%s/%s", chart.Repository, chart.Package)
+		if _, ok := repoSet[name]; !ok {
+			repoSet[name] = nil
+			repositories = append(repositories, &model.Repository{
+				Name: name,
+			})
+		}
 	}
 
 	repositories, err = filter.DoFilterRepositories(repositories, filters)
 	if err != nil {
 		return nil, err
 	}
+	if len(repositories) == 0 {
+		return resources, nil
+	}
 
-	for _, repository := range repositories {
+	if len(repoSet) != len(repositories) {
-		pkgDetail, err := a.client.getHelmPackageDetail(repository.Name)
+		repoSet = map[string]interface{}{}
-		if err != nil {
+		for _, repo := range repositories {
-			log.Errorf("fetch package detail: %v", err)
+			repoSet[repo.Name] = nil
-			return nil, err
 		}
+	}
 
-		var artifacts []*model.Artifact
+	for _, chart := range charts {
-		for _, version := range pkgDetail.AvailableVersions {
+		name := fmt.Sprintf("%s/%s", chart.Repository, chart.Package)
+		if _, ok := repoSet[name]; ok {
 			artifacts = append(artifacts, &model.Artifact{
-				Tags: []string{version.Version},
+				Tags: []string{chart.Version},
 			})
 		}
+	}
 
-		artifacts, err = filter.DoFilterArtifacts(artifacts, filters)
+	artifacts, err = filter.DoFilterArtifacts(artifacts, filters)
-		if err != nil {
+	if err != nil {
-			return nil, err
+		return nil, err
-		}
+	}
-		if len(artifacts) == 0 {
+	if len(artifacts) == 0 {
+		return resources, nil
+	}
+
+	for _, arti := range artifacts {
+		versionSet[arti.Tags[0]] = nil
+	}
+
+	for _, chart := range charts {
+		name := fmt.Sprintf("%s/%s", chart.Repository, chart.Package)
+		if _, ok := repoSet[name]; !ok {
 			continue
 		}
-
+		if _, ok := versionSet[chart.Version]; !ok {
-		for _, artifact := range artifacts {
+			continue
-			resources = append(resources, &model.Resource{
-				Type:     model.ResourceTypeChart,
-				Registry: a.registry,
-				Metadata: &model.ResourceMetadata{
-					Repository: &model.Repository{
-						Name: repository.Name,
-					},
-					Artifacts: []*model.Artifact{artifact},
-				},
-			})
 		}
+		resources = append(resources, &model.Resource{
+			Type:     model.ResourceTypeChart,
+			Registry: a.registry,
+			Metadata: &model.ResourceMetadata{
+				Repository: &model.Repository{
+					Name: name,
+				},
+				Artifacts: []*model.Artifact{
+					{
+						Tags: []string{chart.Version},
+					},
+				},
+			},
+			ExtendedInfo: map[string]interface{}{
+				"contentURL": chart.ContentURL,
+			},
+		})
 	}
 	return resources, nil
 }
 
+// ChartExist will never be used, for this function is only used when endpoint is destination
 func (a *adapter) ChartExist(name, version string) (bool, error) {
 	_, err := a.client.getHelmChartVersion(name, version)
 	if err != nil {
@@ -93,16 +122,11 @@ func (a *adapter) ChartExist(name, version string) (bool, error) {
 	return true, nil
 }
 
-func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) {
+func (a *adapter) DownloadChart(name, version, contentURL string) (io.ReadCloser, error) {
-	chartVersion, err := a.client.getHelmChartVersion(name, version)
+	if len(contentURL) == 0 {
-	if err != nil {
+		return nil, errors.Errorf("empty chart content url, %s:%s", name, version)
-		return nil, err
 	}
-
+	return a.download(contentURL)
-	if len(chartVersion.ContentURL) == 0 {
-		return nil, errors.Errorf("")
-	}
-	return a.download(chartVersion.ContentURL)
 }
 
 func (a *adapter) download(contentURL string) (io.ReadCloser, error) {

src/replication/adapter/artifacthub/client.go

@@ -24,101 +24,6 @@ func newClient(registry *model.Registry) *Client {
 	}
 }
 
-// searchPackages query the artifact package list from artifact hub.
-func (c *Client) searchPackages(kind, offset, limit int, queryString string) (*PackageResponse, error) {
-	request, err := http.NewRequest(http.MethodGet, baseURL+searchPackages(kind, offset, limit, queryString), nil)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := c.do(request)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		msg := &Message{}
-		err = json.Unmarshal(body, msg)
-		if err != nil {
-			msg.Message = string(body)
-		}
-		return nil, fmt.Errorf("search package list error %d: %s", resp.StatusCode, msg.Message)
-	}
-
-	packageResp := &PackageResponse{}
-	err = json.Unmarshal(body, packageResp)
-	if err != nil {
-		return nil, fmt.Errorf("unmarshal package list response error: %v", err)
-	}
-	return packageResp, nil
-}
-
-// getAllPackages gets all of the specific kind of artifact packages from artifact hub.
-func (c *Client) getAllPackages(kind int) (pkgs []*Package, err error) {
-	offset := 0
-	limit := 50
-	shouldContinue := true
-	// todo: rate limit
-	for shouldContinue {
-		pkgResp, err := c.searchPackages(HelmChart, offset, limit, "")
-		if err != nil {
-			return nil, err
-		}
-
-		pkgs = append(pkgs, pkgResp.Data.Packages...)
-		total := pkgResp.Metadata.Total
-		offset = offset + limit
-		if offset >= total {
-			shouldContinue = false
-		}
-	}
-	return pkgs, nil
-}
-
-// getHelmPackageDetail get the chart detail of a helm chart from artifact hub.
-func (c *Client) getHelmPackageDetail(fullName string) (*PackageDetail, error) {
-	request, err := http.NewRequest(http.MethodGet, baseURL+getHelmPackageDetail(fullName), nil)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := c.do(request)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	if resp.StatusCode == http.StatusNotFound {
-		return nil, ErrHTTPNotFound
-	} else if resp.StatusCode != http.StatusOK {
-		msg := &Message{}
-		err = json.Unmarshal(body, msg)
-		if err != nil {
-			msg.Message = string(body)
-		}
-		return nil, fmt.Errorf("fetch package detail error %d: %s", resp.StatusCode, msg.Message)
-	}
-
-	pkgDetail := &PackageDetail{}
-	err = json.Unmarshal(body, pkgDetail)
-	if err != nil {
-		return nil, fmt.Errorf("unmarshal package detail response error: %v", err)
-	}
-
-	return pkgDetail, nil
-}
-
 // getHelmVersion get the package version of a helm chart from artifact hub.
 func (c *Client) getHelmChartVersion(fullName, version string) (*ChartVersion, error) {
 	request, err := http.NewRequest(http.MethodGet, baseURL+getHelmVersion(fullName, version), nil)
@@ -157,6 +62,43 @@ func (c *Client) getHelmChartVersion(fullName, version string) (*ChartVersion, e
 	return chartVersion, nil
 }
 
+// getReplicationInfo gets the brief info of all helm chart from artifact hub.
+// see https://github.com/artifacthub/hub/issues/997
+func (c *Client) getReplicationInfo() ([]*ChartInfo, error) {
+	request, err := http.NewRequest(http.MethodGet, baseURL+getReplicationInfo, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := c.do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		msg := &Message{}
+		err = json.Unmarshal(body, msg)
+		if err != nil {
+			msg.Message = string(body)
+		}
+		return nil, fmt.Errorf("get chart replication info error %d: %s", resp.StatusCode, msg.Message)
+	}
+
+	var chartInfo []*ChartInfo
+	err = json.Unmarshal(body, &chartInfo)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal chart replication info error: %v", err)
+	}
+
+	return chartInfo, nil
+}
+
 func (c *Client) checkHealthy() error {
 	request, err := http.NewRequest(http.MethodGet, baseURL, nil)
 	if err != nil {

src/replication/adapter/artifacthub/consts.go

@@ -6,7 +6,8 @@ import (
 )
 
 const (
-	baseURL = "https://artifacthub.io"
+	baseURL            = "https://artifacthub.io"
+	getReplicationInfo = "/api/v1/harborReplication"
 )
 
 const (

src/replication/adapter/artifacthub/model.go

@@ -13,9 +13,10 @@ type PackageData struct {
 
 // Package ...
 type Package struct {
-	PackageID  string      `json:"package_id"`
+	PackageID      string      `json:"package_id"`
-	Name       string      `json:"name"`
+	Name           string      `json:"name"`
-	Repository *Repository `json:"repository"`
+	NormalizedName string      `json:"normalized_name"`
+	Repository     *Repository `json:"repository"`
 }
 
 // Repository ...
@@ -29,6 +30,7 @@ type Repository struct {
 type PackageDetail struct {
 	PackageID         string           `json:"package_id"`
 	Name              string           `json:"name"`
+	NormalizedName    string           `json:"normalized_name"`
 	Version           string           `json:"version"`
 	AppVersion        string           `json:"app_version"`
 	Repository        RepositoryDetail `json:"repository"`
@@ -70,3 +72,11 @@ type Metadata struct {
 type Message struct {
 	Message string `json:"message"`
 }
+
+// ChartInfo ...
+type ChartInfo struct {
+	Repository string `json:"repository"`
+	Package    string `json:"package"`
+	Version    string `json:"version"`
+	ContentURL string `json:"url"`
+}

src/replication/adapter/harbor/base/chart_registry.go

@@ -144,7 +144,7 @@ func (a *Adapter) getChartInfo(name, version string) (*chartVersionDetail, error
 }
 
 // DownloadChart downloads the specific chart
-func (a *Adapter) DownloadChart(name, version string) (io.ReadCloser, error) {
+func (a *Adapter) DownloadChart(name, version, contentURL string) (io.ReadCloser, error) {
 	info, err := a.getChartInfo(name, version)
 	if err != nil {
 		return nil, err

src/replication/adapter/harbor/base/chart_registry_test.go

@@ -149,7 +149,7 @@ func TestDownloadChart(t *testing.T) {
 	}
 	adapter, err := New(registry)
 	require.Nil(t, err)
-	_, err = adapter.DownloadChart("library/harbor", "1.0")
+	_, err = adapter.DownloadChart("library/harbor", "1.0", "")
 	require.Nil(t, err)
 }
 

src/replication/adapter/helmhub/chart_registry.go

@@ -101,7 +101,7 @@ func (a *adapter) ChartExist(name, version string) (bool, error) {
 	return false, nil
 }
 
-func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) {
+func (a *adapter) DownloadChart(name, version, contentURL string) (io.ReadCloser, error) {
 	versionList, err := a.client.fetchChartDetail(name)
 	if err != nil {
 		return nil, err

src/replication/adapter/tencentcr/chart_registry.go

@@ -146,7 +146,7 @@ func (a *adapter) getChartInfo(name, version string) (info *tcrChartVersionDetai
 	return
 }
 
-func (a *adapter) DownloadChart(name, version string) (rc io.ReadCloser, err error) {
+func (a *adapter) DownloadChart(name, version, contentURL string) (rc io.ReadCloser, err error) {
 	var info *tcrChartVersionDetail
 	info, err = a.getChartInfo(name, version)
 	if err != nil {

src/replication/transfer/chart/transfer.go

@@ -37,8 +37,9 @@ func factory(logger trans.Logger, stopFunc trans.StopFunc) (trans.Transfer, erro
 }
 
 type chart struct {
-	name    string
+	name       string
-	version string
+	version    string
+	contentURL string
 }
 
 type transfer struct {
@@ -62,9 +63,15 @@ func (t *transfer) Transfer(src *model.Resource, dst *model.Resource) error {
 		})
 	}
 
+	var contentURL string
+	if len(src.ExtendedInfo) > 0 {
+		contentURL = src.ExtendedInfo["contentURL"].(string)
+	}
+
 	srcChart := &chart{
-		name:    src.Metadata.Repository.Name,
+		name:       src.Metadata.Repository.Name,
-		version: src.Metadata.Artifacts[0].Tags[0],
+		version:    src.Metadata.Artifacts[0].Tags[0],
+		contentURL: contentURL,
 	}
 	dstChart := &chart{
 		name:    dst.Metadata.Repository.Name,
@@ -151,7 +158,7 @@ func (t *transfer) copy(src, dst *chart, override bool) error {
 	}
 
 	// copy the chart between the source and destination registries
-	chart, err := t.src.DownloadChart(src.name, src.version)
+	chart, err := t.src.DownloadChart(src.name, src.version, src.contentURL)
 	if err != nil {
 		t.logger.Errorf("failed to download the chart %s:%s: %v", src.name, src.version, err)
 		return err

src/replication/transfer/chart/transfer_test.go

@@ -45,7 +45,7 @@ func (f *fakeRegistry) FetchCharts(filters []*model.Filter) ([]*model.Resource,
 func (f *fakeRegistry) ChartExist(name, version string) (bool, error) {
 	return true, nil
 }
-func (f *fakeRegistry) DownloadChart(name, version string) (io.ReadCloser, error) {
+func (f *fakeRegistry) DownloadChart(name, version, contentURL string) (io.ReadCloser, error) {
 	r := ioutil.NopCloser(bytes.NewReader([]byte{'a'}))
 	return r, nil
 }