Merge pull request #13618 from mmpei/official-feature-artifact-hub

Support replicate from artifact hub

src/jobservice/job/impl/replication/replication.go

@@ -52,6 +52,8 @@ import (
 	_ "github.com/goharbor/harbor/src/replication/adapter/gitlab"
 	// register the DTR adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/dtr"
+	// register the Artifact Hub adapter
+	_ "github.com/goharbor/harbor/src/replication/adapter/artifacthub"
 )
 
 // Replication implements the job interface

src/replication/adapter/artifacthub/adapter.go

@@ -0,0 +1,107 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package artifacthub
+
+import (
+	"errors"
+
+	"github.com/goharbor/harbor/src/lib/log"
+	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/model"
+)
+
+func init() {
+	if err := adp.RegisterFactory(model.RegistryTypeArtifactHub, new(factory)); err != nil {
+		log.Errorf("failed to register factory for %s: %v", model.RegistryTypeArtifactHub, err)
+		return
+	}
+	log.Infof("the factory for adapter %s registered", model.RegistryTypeArtifactHub)
+}
+
+type factory struct {
+}
+
+// Create ...
+func (f *factory) Create(r *model.Registry) (adp.Adapter, error) {
+	return newAdapter(r)
+}
+
+// AdapterPattern ...
+func (f *factory) AdapterPattern() *model.AdapterPattern {
+	return &model.AdapterPattern{
+		EndpointPattern: &model.EndpointPattern{
+			EndpointType: model.EndpointPatternTypeFix,
+			Endpoints: []*model.Endpoint{
+				{
+					Key:   "artifacthub.io",
+					Value: "https://artifacthub.io",
+				},
+			},
+		},
+	}
+}
+
+var (
+	_ adp.Adapter       = (*adapter)(nil)
+	_ adp.ChartRegistry = (*adapter)(nil)
+)
+
+type adapter struct {
+	registry *model.Registry
+	client   *Client
+}
+
+func newAdapter(registry *model.Registry) (*adapter, error) {
+	return &adapter{
+		registry: registry,
+		client:   newClient(registry),
+	}, nil
+}
+
+func (a *adapter) Info() (*model.RegistryInfo, error) {
+	return &model.RegistryInfo{
+		Type: model.RegistryTypeArtifactHub,
+		SupportedResourceTypes: []model.ResourceType{
+			model.ResourceTypeChart,
+		},
+		SupportedResourceFilters: []*model.FilterStyle{
+			{
+				Type:  model.FilterTypeName,
+				Style: model.FilterStyleTypeText,
+			},
+			{
+				Type:  model.FilterTypeTag,
+				Style: model.FilterStyleTypeText,
+			},
+		},
+		SupportedTriggers: []model.TriggerType{
+			model.TriggerTypeManual,
+			model.TriggerTypeScheduled,
+		},
+	}, nil
+}
+
+func (a *adapter) PrepareForPush(resources []*model.Resource) error {
+	return errors.New("not supported")
+}
+
+// HealthCheck checks health status of a registry
+func (a *adapter) HealthCheck() (model.HealthStatus, error) {
+	err := a.client.checkHealthy()
+	if err == nil {
+		return model.Healthy, nil
+	}
+	return model.Unhealthy, err
+}

src/replication/adapter/artifacthub/adapter_test.go

@@ -0,0 +1,108 @@
+package artifacthub
+
+import (
+	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestAdapter_NewAdapter(t *testing.T) {
+	factory, err := adp.GetFactory("BadName")
+	assert.Nil(t, factory)
+	assert.NotNil(t, err)
+
+	factory, err = adp.GetFactory(model.RegistryTypeArtifactHub)
+	assert.Nil(t, err)
+	assert.NotNil(t, factory)
+
+	adapter, err := newAdapter(&model.Registry{
+		Type: model.RegistryTypeArtifactHub,
+		URL:  "https://artifacthub.io",
+	})
+	assert.Nil(t, err)
+	assert.NotNil(t, adapter)
+}
+
+func TestAdapter_Info(t *testing.T) {
+	a, _ := newAdapter(&model.Registry{
+		Type: model.RegistryTypeArtifactHub,
+		URL:  "https://artifacthub.io",
+	})
+	info, err := a.Info()
+	assert.Nil(t, err)
+	assert.NotNil(t, info)
+
+	assert.EqualValues(t, model.RegistryTypeArtifactHub, info.Type)
+	assert.EqualValues(t, 1, len(info.SupportedResourceTypes))
+	assert.EqualValues(t, model.ResourceTypeChart, info.SupportedResourceTypes[0])
+}
+
+func TestAdapter_HealthCheck(t *testing.T) {
+	a, _ := newAdapter(&model.Registry{
+		Type: model.RegistryTypeArtifactHub,
+		URL:  "https://artifacthub.io",
+	})
+	h, err := a.HealthCheck()
+	assert.Nil(t, err)
+	assert.EqualValues(t, model.Healthy, h)
+}
+
+func TestAdapter_PrepareForPush(t *testing.T) {
+	a, _ := newAdapter(&model.Registry{
+		Type: model.RegistryTypeArtifactHub,
+		URL:  "https://artifacthub.io",
+	})
+	err := a.PrepareForPush(nil)
+	assert.NotNil(t, err)
+}
+
+func TestAdapter_ChartExist(t *testing.T) {
+	a, _ := newAdapter(&model.Registry{
+		Type: model.RegistryTypeArtifactHub,
+		URL:  "https://artifacthub.io",
+	})
+
+	b, err := a.ChartExist("harbor/harbor", "1.5.0")
+	assert.Nil(t, err)
+	assert.True(t, b)
+
+	b, err = a.ChartExist("harbor/not-exists", "1.5.0")
+	assert.Nil(t, err)
+	assert.False(t, b)
+
+	b, err = a.ChartExist("harbor/harbor", "not-exists")
+	assert.Nil(t, err)
+	assert.False(t, b)
+}
+
+func TestAdapter_DownloadChart(t *testing.T) {
+	a, _ := newAdapter(&model.Registry{
+		Type: model.RegistryTypeArtifactHub,
+		URL:  "https://artifacthub.io",
+	})
+
+	data, err := a.DownloadChart("harbor/harbor", "1.5.0")
+	assert.Nil(t, err)
+	assert.NotNil(t, data)
+}
+
+func TestAdapter_DeleteChart(t *testing.T) {
+	a, _ := newAdapter(&model.Registry{
+		Type: model.RegistryTypeArtifactHub,
+		URL:  "https://artifacthub.io",
+	})
+
+	err := a.DeleteChart("harbor/harbor", "1.5.0")
+	assert.NotNil(t, err)
+}
+
+func TestAdapter_UploadChart(t *testing.T) {
+	a, _ := newAdapter(&model.Registry{
+		Type: model.RegistryTypeArtifactHub,
+		URL:  "https://artifacthub.io",
+	})
+
+	err := a.UploadChart("harbor/harbor", "1.5.0", nil)
+	assert.NotNil(t, err)
+}

src/replication/adapter/artifacthub/chart_registry.go

@@ -0,0 +1,133 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package artifacthub
+
+import (
+	"fmt"
+	"github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/replication/filter"
+	"github.com/goharbor/harbor/src/replication/model"
+	"io"
+	"io/ioutil"
+	"net/http"
+)
+
+func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) {
+	pkgs, err := a.client.getAllPackages(HelmChart)
+	if err != nil {
+		return nil, err
+	}
+
+	resources := []*model.Resource{}
+	var repositories []*model.Repository
+	for _, pkg := range pkgs {
+		repositories = append(repositories, &model.Repository{
+			Name: fmt.Sprintf("%s/%s", pkg.Repository.Name, pkg.Name),
+		})
+	}
+
+	repositories, err = filter.DoFilterRepositories(repositories, filters)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, repository := range repositories {
+		pkgDetail, err := a.client.getHelmPackageDetail(repository.Name)
+		if err != nil {
+			log.Errorf("fetch package detail: %v", err)
+			return nil, err
+		}
+
+		var artifacts []*model.Artifact
+		for _, version := range pkgDetail.AvailableVersions {
+			artifacts = append(artifacts, &model.Artifact{
+				Tags: []string{version.Version},
+			})
+		}
+
+		artifacts, err = filter.DoFilterArtifacts(artifacts, filters)
+		if err != nil {
+			return nil, err
+		}
+		if len(artifacts) == 0 {
+			continue
+		}
+
+		for _, artifact := range artifacts {
+			resources = append(resources, &model.Resource{
+				Type:     model.ResourceTypeChart,
+				Registry: a.registry,
+				Metadata: &model.ResourceMetadata{
+					Repository: &model.Repository{
+						Name: repository.Name,
+					},
+					Artifacts: []*model.Artifact{artifact},
+				},
+			})
+		}
+	}
+	return resources, nil
+}
+
+func (a *adapter) ChartExist(name, version string) (bool, error) {
+	_, err := a.client.getHelmChartVersion(name, version)
+	if err != nil {
+		if err == ErrHTTPNotFound {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) {
+	chartVersion, err := a.client.getHelmChartVersion(name, version)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(chartVersion.ContentURL) == 0 {
+		return nil, errors.Errorf("")
+	}
+	return a.download(chartVersion.ContentURL)
+}
+
+func (a *adapter) download(contentURL string) (io.ReadCloser, error) {
+	req, err := http.NewRequest(http.MethodGet, contentURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := a.client.do(req)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+		return nil, fmt.Errorf("failed to download the chart %s: %d %s", contentURL, resp.StatusCode, string(body))
+	}
+	return resp.Body, nil
+}
+
+func (a *adapter) UploadChart(name, version string, chart io.Reader) error {
+	return errors.New("not supported")
+}
+
+func (a *adapter) DeleteChart(name, version string) error {
+	return errors.New("not supported")
+}

src/replication/adapter/artifacthub/client.go

@@ -0,0 +1,182 @@
+package artifacthub
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/goharbor/harbor/src/replication/util"
+	"io/ioutil"
+	"net/http"
+)
+
+// Client is a client to interact with Artifact Hub
+type Client struct {
+	httpClient *http.Client
+}
+
+// newClient creates a new ArtifactHub client.
+func newClient(registry *model.Registry) *Client {
+	return &Client{
+		httpClient: &http.Client{
+			Transport: util.GetHTTPTransport(registry.Insecure),
+		},
+	}
+}
+
+// searchPackages query the artifact package list from artifact hub.
+func (c *Client) searchPackages(kind, offset, limit int, queryString string) (*PackageResponse, error) {
+	request, err := http.NewRequest(http.MethodGet, baseURL+searchPackages(kind, offset, limit, queryString), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := c.do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		msg := &Message{}
+		err = json.Unmarshal(body, msg)
+		if err != nil {
+			msg.Message = string(body)
+		}
+		return nil, fmt.Errorf("search package list error %d: %s", resp.StatusCode, msg.Message)
+	}
+
+	packageResp := &PackageResponse{}
+	err = json.Unmarshal(body, packageResp)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal package list response error: %v", err)
+	}
+	return packageResp, nil
+}
+
+// getAllPackages gets all of the specific kind of artifact packages from artifact hub.
+func (c *Client) getAllPackages(kind int) (pkgs []*Package, err error) {
+	offset := 0
+	limit := 50
+	shouldContinue := true
+	// todo: rate limit
+	for shouldContinue {
+		pkgResp, err := c.searchPackages(HelmChart, offset, limit, "")
+		if err != nil {
+			return nil, err
+		}
+
+		pkgs = append(pkgs, pkgResp.Data.Packages...)
+		total := pkgResp.Metadata.Total
+		offset = offset + limit
+		if offset >= total {
+			shouldContinue = false
+		}
+	}
+	return pkgs, nil
+}
+
+// getHelmPackageDetail get the chart detail of a helm chart from artifact hub.
+func (c *Client) getHelmPackageDetail(fullName string) (*PackageDetail, error) {
+	request, err := http.NewRequest(http.MethodGet, baseURL+getHelmPackageDetail(fullName), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := c.do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode == http.StatusNotFound {
+		return nil, ErrHTTPNotFound
+	} else if resp.StatusCode != http.StatusOK {
+		msg := &Message{}
+		err = json.Unmarshal(body, msg)
+		if err != nil {
+			msg.Message = string(body)
+		}
+		return nil, fmt.Errorf("fetch package detail error %d: %s", resp.StatusCode, msg.Message)
+	}
+
+	pkgDetail := &PackageDetail{}
+	err = json.Unmarshal(body, pkgDetail)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal package detail response error: %v", err)
+	}
+
+	return pkgDetail, nil
+}
+
+// getHelmVersion get the package version of a helm chart from artifact hub.
+func (c *Client) getHelmChartVersion(fullName, version string) (*ChartVersion, error) {
+	request, err := http.NewRequest(http.MethodGet, baseURL+getHelmVersion(fullName, version), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := c.do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode == http.StatusNotFound {
+		return nil, ErrHTTPNotFound
+	} else if resp.StatusCode != http.StatusOK {
+		msg := &Message{}
+		err = json.Unmarshal(body, msg)
+		if err != nil {
+			msg.Message = string(body)
+		}
+		return nil, fmt.Errorf("fetch chart version error %d: %s", resp.StatusCode, msg.Message)
+	}
+
+	chartVersion := &ChartVersion{}
+	err = json.Unmarshal(body, chartVersion)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal chart version response error: %v", err)
+	}
+
+	return chartVersion, nil
+}
+
+func (c *Client) checkHealthy() error {
+	request, err := http.NewRequest(http.MethodGet, baseURL, nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := c.do(request)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	ioutil.ReadAll(resp.Body)
+	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
+		return nil
+	}
+	return errors.New("artifact hub is unhealthy")
+}
+
+// do work as a proxy of Do function from net.http
+func (c *Client) do(req *http.Request) (*http.Response, error) {
+	return c.httpClient.Do(req)
+}

src/replication/adapter/artifacthub/consts.go

@@ -0,0 +1,41 @@
+package artifacthub
+
+import (
+	"errors"
+	"fmt"
+)
+
+const (
+	baseURL = "https://artifacthub.io"
+)
+
+const (
+	// HelmChart represents the kind of helm chart in artifact hub
+	HelmChart = iota
+	// FalcoRules represents the kind of falco rules in artifact hub
+	FalcoRules
+	// OPAPolicies represents the kind of OPA policies in artifact hub
+	OPAPolicies
+	// OLMOperators represents the kind of OLM operators in artifact hub
+	OLMOperators
+)
+
+// ErrHTTPNotFound defines the return error when receiving 404 response code
+var ErrHTTPNotFound = errors.New("Not Found")
+
+func searchPackages(kind, offset, limit int, queryString string) string {
+	if len(queryString) == 0 {
+		return fmt.Sprintf("/api/v1/packages/search?kind=%d&limit=%d&offset=%d",
+			kind, limit, offset)
+	}
+	return fmt.Sprintf("/api/v1/packages/search?kind=%d&limit=%d&offset=%d&ts_query_web=%s",
+		kind, limit, offset, queryString)
+}
+
+func getHelmPackageDetail(fullName string) string {
+	return fmt.Sprintf("/api/v1/packages/helm/%s", fullName)
+}
+
+func getHelmVersion(fullName, version string) string {
+	return fmt.Sprintf("/api/v1/packages/helm/%s/%s", fullName, version)
+}

src/replication/adapter/artifacthub/model.go

@@ -0,0 +1,72 @@
+package artifacthub
+
+// PackageResponse ...
+type PackageResponse struct {
+	Data     PackageData `json:"data"`
+	Metadata Metadata    `json:"metadata"`
+}
+
+// PackageData ...
+type PackageData struct {
+	Packages []*Package `json:"packages"`
+}
+
+// Package ...
+type Package struct {
+	PackageID  string      `json:"package_id"`
+	Name       string      `json:"name"`
+	Repository *Repository `json:"repository"`
+}
+
+// Repository ...
+type Repository struct {
+	Kind         int    `json:"kind"`
+	Name         string `json:"name"`
+	RepositoryID string `json:"repository_id"`
+}
+
+// PackageDetail ...
+type PackageDetail struct {
+	PackageID         string           `json:"package_id"`
+	Name              string           `json:"name"`
+	Version           string           `json:"version"`
+	AppVersion        string           `json:"app_version"`
+	Repository        RepositoryDetail `json:"repository"`
+	AvailableVersions []*Version       `json:"available_versions,omitempty"`
+}
+
+// RepositoryDetail ...
+type RepositoryDetail struct {
+	URL               string `json:"url"`
+	Kind              int    `json:"kind"`
+	Name              string `json:"name"`
+	RepositoryID      string `json:"repository_id"`
+	VerifiedPublisher bool   `json:"verified_publisher"`
+	Official          bool   `json:"official"`
+	Private           bool   `json:"private"`
+}
+
+// Version ...
+type Version struct {
+	Version string `json:"version"`
+}
+
+// ChartVersion ...
+type ChartVersion struct {
+	PackageID  string `json:"package_id"`
+	Name       string `json:"name"`
+	Version    string `json:"version"`
+	ContentURL string `json:"content_url"`
+}
+
+// Metadata ...
+type Metadata struct {
+	Limit  int `json:"limit"`
+	Offset int `json:"offset"`
+	Total  int `json:"total"`
+}
+
+// Message ...
+type Message struct {
+	Message string `json:"message"`
+}

src/replication/model/registry.go

@@ -33,7 +33,8 @@ const (
 	RegistryTypeGitLab           RegistryType = "gitlab"
 	RegistryTypeDTR              RegistryType = "dtr"
 
-	RegistryTypeHelmHub RegistryType = "helm-hub"
+	RegistryTypeHelmHub     RegistryType = "helm-hub"
+	RegistryTypeArtifactHub RegistryType = "artifact-hub"
 
 	FilterStyleTypeText  = "input"
 	FilterStyleTypeRadio = "radio"

src/replication/replication.go

@@ -57,6 +57,8 @@ import (
 	_ "github.com/goharbor/harbor/src/replication/adapter/gitlab"
 	// register the DTR adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/dtr"
+	// register the Artifact Hub adapter
+	_ "github.com/goharbor/harbor/src/replication/adapter/artifacthub"
 )
 
 var (