Merge pull request #8244 from mmpei/7980-chartReplication

Support helm chart replication from helmhub

src/replication/adapter/helmhub/adapter.go

@@ -0,0 +1,80 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package helmhub
+
+import (
+	"errors"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/model"
+)
+
+func init() {
+	if err := adp.RegisterFactory(model.RegistryTypeHelmHub, func(registry *model.Registry) (adp.Adapter, error) {
+		return newAdapter(registry)
+	}); err != nil {
+		log.Errorf("failed to register factory for %s: %v", model.RegistryTypeHelmHub, err)
+		return
+	}
+	log.Infof("the factory for adapter %s registered", model.RegistryTypeHelmHub)
+}
+
+type adapter struct {
+	registry *model.Registry
+	client   *Client
+}
+
+func newAdapter(registry *model.Registry) (*adapter, error) {
+	return &adapter{
+		registry: registry,
+		client:   NewClient(registry),
+	}, nil
+}
+
+func (a *adapter) Info() (*model.RegistryInfo, error) {
+	return &model.RegistryInfo{
+		Type: model.RegistryTypeHelmHub,
+		SupportedResourceTypes: []model.ResourceType{
+			model.ResourceTypeChart,
+		},
+		SupportedResourceFilters: []*model.FilterStyle{
+			{
+				Type:  model.FilterTypeName,
+				Style: model.FilterStyleTypeText,
+			},
+			{
+				Type:  model.FilterTypeTag,
+				Style: model.FilterStyleTypeText,
+			},
+		},
+		SupportedTriggers: []model.TriggerType{
+			model.TriggerTypeManual,
+			model.TriggerTypeScheduled,
+		},
+	}, nil
+}
+
+func (a *adapter) PrepareForPush(resources []*model.Resource) error {
+	return errors.New("not supported")
+}
+
+// HealthCheck checks health status of a registry
+func (a *adapter) HealthCheck() (model.HealthStatus, error) {
+	err := a.client.checkHealthy()
+	if err == nil {
+		return model.Healthy, nil
+	}
+	return model.Unhealthy, err
+}

src/replication/adapter/helmhub/adapter_test.go

@@ -0,0 +1,44 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package helmhub
+
+import (
+	"testing"
+
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestInfo(t *testing.T) {
+	adapter := &adapter{}
+	info, err := adapter.Info()
+	require.Nil(t, err)
+	require.Equal(t, 1, len(info.SupportedResourceTypes))
+	assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[0])
+}
+
+func TestPrepareForPush(t *testing.T) {
+	adapter := &adapter{}
+	err := adapter.PrepareForPush(nil)
+	require.NotNil(t, err)
+}
+
+func TestHealthCheck(t *testing.T) {
+	adapter, _ := newAdapter(nil)
+	status, err := adapter.HealthCheck()
+	require.Equal(t, model.Healthy, string(status))
+	require.Nil(t, err)
+}

src/replication/adapter/helmhub/chart.go

@@ -0,0 +1,44 @@
+package helmhub
+
+type chart struct {
+	ID   string `json:"id"`
+	Type string `json:"type"`
+}
+
+type chartList struct {
+	Data []*chart `json:"data"`
+}
+
+type chartAttributes struct {
+	Version string   `json:"version"`
+	URLs    []string `json:"urls"`
+}
+
+type chartRepo struct {
+	Name string `json:"name"`
+	URL  string `json:"url"`
+}
+
+type chartData struct {
+	Name string     `json:"name"`
+	Repo *chartRepo `json:"repo"`
+}
+
+type chartInfo struct {
+	Data *chartData `json:"data"`
+}
+
+type chartRelationships struct {
+	Chart *chartInfo `json:"chart"`
+}
+
+type chartVersion struct {
+	ID            string              `json:"id"`
+	Type          string              `json:"type"`
+	Attributes    *chartAttributes    `json:"attributes"`
+	Relationships *chartRelationships `json:"relationships"`
+}
+
+type chartVersionList struct {
+	Data []*chartVersion `json:"data"`
+}

src/replication/adapter/helmhub/chart_registry.go

@@ -0,0 +1,146 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package helmhub
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/goharbor/harbor/src/common/utils/log"
+	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/pkg/errors"
+)
+
+func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) {
+	charts, err := a.client.fetchCharts()
+	if err != nil {
+		return nil, err
+	}
+
+	resources := []*model.Resource{}
+	repositories := []*adp.Repository{}
+	for _, chart := range charts.Data {
+		repository := &adp.Repository{
+			ResourceType: string(model.ResourceTypeChart),
+			Name:         chart.ID,
+		}
+		repositories = append(repositories, repository)
+	}
+
+	for _, filter := range filters {
+		if err = filter.DoFilter(&repositories); err != nil {
+			return nil, err
+		}
+	}
+
+	for _, repository := range repositories {
+		versionList, err := a.client.fetchChartDetail(repository.Name)
+		if err != nil {
+			log.Errorf("fetch chart detail: %v", err)
+			return nil, err
+		}
+
+		vTags := []*adp.VTag{}
+		for _, version := range versionList.Data {
+			vTags = append(vTags, &adp.VTag{
+				Name:         version.Attributes.Version,
+				ResourceType: string(model.ResourceTypeChart),
+			})
+		}
+
+		for _, filter := range filters {
+			if err = filter.DoFilter(&vTags); err != nil {
+				return nil, err
+			}
+		}
+
+		for _, vTag := range vTags {
+			resources = append(resources, &model.Resource{
+				Type:     model.ResourceTypeChart,
+				Registry: a.registry,
+				Metadata: &model.ResourceMetadata{
+					Repository: &model.Repository{
+						Name: repository.Name,
+					},
+					Vtags: []string{vTag.Name},
+				},
+			})
+		}
+	}
+	return resources, nil
+}
+
+func (a *adapter) ChartExist(name, version string) (bool, error) {
+	versionList, err := a.client.fetchChartDetail(name)
+	if err != nil {
+		if err == ErrHTTPNotFound {
+			return false, nil
+		}
+		return false, err
+	}
+
+	for _, v := range versionList.Data {
+		if v.Attributes.Version == version {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) {
+	versionList, err := a.client.fetchChartDetail(name)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, v := range versionList.Data {
+		if v.Attributes.Version == version {
+			return a.download(v)
+		}
+	}
+	return nil, errors.New("chart not found")
+}
+
+func (a *adapter) download(version *chartVersion) (io.ReadCloser, error) {
+	if len(version.Attributes.URLs) == 0 || len(version.Attributes.URLs[0]) == 0 {
+		return nil, fmt.Errorf("cannot got the download url for chart %s", version.ID)
+	}
+
+	url := strings.ToLower(version.Attributes.URLs[0])
+	if !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) {
+		url = fmt.Sprintf("%s/charts/%s", version.Relationships.Chart.Data.Repo.URL, url)
+	}
+
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := a.client.do(req)
+	if err != nil {
+		return nil, err
+	}
+	return resp.Body, nil
+}
+
+func (a *adapter) UploadChart(name, version string, chart io.Reader) error {
+	return errors.New("not supported")
+}
+
+func (a *adapter) DeleteChart(name, version string) error {
+	return errors.New("not supported")
+}

src/replication/adapter/helmhub/chart_registry_test.go

@@ -0,0 +1,94 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package helmhub
+
+import (
+	"testing"
+
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFetchCharts(t *testing.T) {
+	adapter, err := newAdapter(nil)
+	require.Nil(t, err)
+	// filter 1
+	filters := []*model.Filter{
+		{
+			Type:  model.FilterTypeName,
+			Value: "k*/*",
+		},
+	}
+	resources, err := adapter.FetchCharts(filters)
+	require.Nil(t, err)
+	assert.NotZero(t, len(resources))
+	assert.Equal(t, model.ResourceTypeChart, resources[0].Type)
+	assert.Equal(t, 1, len(resources[0].Metadata.Vtags))
+	assert.NotNil(t, resources[0].Metadata.Vtags[0])
+	// filter 2
+	filters = []*model.Filter{
+		{
+			Type:  model.FilterTypeName,
+			Value: "harbor/*",
+		},
+	}
+	resources, err = adapter.FetchCharts(filters)
+	require.Nil(t, err)
+	assert.NotZero(t, len(resources))
+	assert.Equal(t, model.ResourceTypeChart, resources[0].Type)
+	assert.Equal(t, "harbor/harbor", resources[0].Metadata.Repository.Name)
+	assert.Equal(t, 1, len(resources[0].Metadata.Vtags))
+	assert.NotNil(t, resources[0].Metadata.Vtags[0])
+}
+
+func TestChartExist(t *testing.T) {
+	adapter, err := newAdapter(nil)
+	require.Nil(t, err)
+	exist, err := adapter.ChartExist("harbor/harbor", "1.0.0")
+	require.Nil(t, err)
+	require.True(t, exist)
+}
+
+func TestChartExist2(t *testing.T) {
+	adapter, err := newAdapter(nil)
+	require.Nil(t, err)
+	exist, err := adapter.ChartExist("goharbor/harbor", "1.0.0")
+	require.Nil(t, err)
+	require.False(t, exist)
+
+	exist, err = adapter.ChartExist("harbor/harbor", "1.0.100")
+	require.Nil(t, err)
+	require.False(t, exist)
+}
+
+func TestDownloadChart(t *testing.T) {
+	adapter, err := newAdapter(nil)
+	require.Nil(t, err)
+	_, err = adapter.DownloadChart("harbor/harbor", "1.0.0")
+	require.Nil(t, err)
+}
+
+func TestUploadChart(t *testing.T) {
+	adapter := &adapter{}
+	err := adapter.UploadChart("library/harbor", "1.0", nil)
+	require.NotNil(t, err)
+}
+
+func TestDeleteChart(t *testing.T) {
+	adapter := &adapter{}
+	err := adapter.DeleteChart("library/harbor", "1.0")
+	require.NotNil(t, err)
+}

src/replication/adapter/helmhub/client.go

@@ -0,0 +1,116 @@
+package helmhub
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/goharbor/harbor/src/replication/util"
+	"github.com/pkg/errors"
+	"io/ioutil"
+	"net/http"
+)
+
+// ErrHTTPNotFound defines the return error when receiving 404 response code
+var ErrHTTPNotFound = errors.New("Not Found")
+
+// Client is a client to interact with HelmHub
+type Client struct {
+	client *http.Client
+}
+
+// NewClient creates a new HelmHub client.
+func NewClient(registry *model.Registry) *Client {
+	return &Client{
+		client: &http.Client{
+			Transport: util.GetHTTPTransport(false),
+		},
+	}
+}
+
+// fetchCharts fetches the chart list from helm hub.
+func (c *Client) fetchCharts() (*chartList, error) {
+	request, err := http.NewRequest(http.MethodGet, baseURL+listCharts, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := c.client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("fetch chart list error %d: %s", resp.StatusCode, string(body))
+	}
+
+	list := &chartList{}
+	err = json.Unmarshal(body, list)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal chart list response error: %v", err)
+	}
+
+	return list, nil
+}
+
+// fetchChartDetail fetches the chart detail of a chart from helm hub.
+func (c *Client) fetchChartDetail(chartName string) (*chartVersionList, error) {
+	request, err := http.NewRequest(http.MethodGet, baseURL+listVersions(chartName), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := c.client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode == http.StatusNotFound {
+		return nil, ErrHTTPNotFound
+	} else if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("fetch chart detail error %d: %s", resp.StatusCode, string(body))
+	}
+
+	list := &chartVersionList{}
+	err = json.Unmarshal(body, list)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal chart detail response error: %v", err)
+	}
+
+	return list, nil
+}
+
+func (c *Client) checkHealthy() error {
+	request, err := http.NewRequest(http.MethodGet, baseURL, nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := c.client.Do(request)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	ioutil.ReadAll(resp.Body)
+	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
+		return nil
+	}
+	return errors.New("helm hub is unhealthy")
+}
+
+// do work as a proxy of Do function from net.http
+func (c *Client) do(req *http.Request) (*http.Response, error) {
+	return c.client.Do(req)
+}

src/replication/adapter/helmhub/consts.go

@@ -0,0 +1,12 @@
+package helmhub
+
+import "fmt"
+
+const (
+	baseURL    = "https://hub.helm.sh"
+	listCharts = "/api/chartsvc/v1/charts"
+)
+
+func listVersions(chartName string) string {
+	return fmt.Sprintf("/api/chartsvc/v1/charts/%s/versions", chartName)
+}

src/replication/model/registry.go

@@ -30,6 +30,8 @@ const (
 	RegistryTypeAwsEcr         RegistryType = "aws-ecr"
 	RegistryTypeAzureAcr       RegistryType = "azure-acr"
 
+	RegistryTypeHelmHub RegistryType = "helm-hub"
+
 	FilterStyleTypeText  = "input"
 	FilterStyleTypeRadio = "radio"
 	FilterStyleTypeList  = "list"