From 4946c7bba77217c4aeb85b38e2c936b22d53b35c Mon Sep 17 00:00:00 2001 From: peimingming Date: Tue, 9 Jul 2019 11:15:29 +0800 Subject: [PATCH 1/3] Add helm chart replication from helmhub Signed-off-by: peimingming --- src/replication/adapter/helmhub/adapter.go | 76 +++++++++ .../adapter/helmhub/adapter_test.go | 44 ++++++ src/replication/adapter/helmhub/chart.go | 44 ++++++ .../adapter/helmhub/chart_registry.go | 145 ++++++++++++++++++ .../adapter/helmhub/chart_registry_test.go | 94 ++++++++++++ src/replication/adapter/helmhub/client.go | 97 ++++++++++++ src/replication/adapter/helmhub/consts.go | 12 ++ src/replication/model/registry.go | 2 + 8 files changed, 514 insertions(+) create mode 100644 src/replication/adapter/helmhub/adapter.go create mode 100644 src/replication/adapter/helmhub/adapter_test.go create mode 100644 src/replication/adapter/helmhub/chart.go create mode 100644 src/replication/adapter/helmhub/chart_registry.go create mode 100644 src/replication/adapter/helmhub/chart_registry_test.go create mode 100644 src/replication/adapter/helmhub/client.go create mode 100644 src/replication/adapter/helmhub/consts.go diff --git a/src/replication/adapter/helmhub/adapter.go b/src/replication/adapter/helmhub/adapter.go new file mode 100644 index 000000000..28c855dc8 --- /dev/null +++ b/src/replication/adapter/helmhub/adapter.go @@ -0,0 +1,76 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package helmhub + +import ( + "errors" + "github.com/goharbor/harbor/src/common/utils/log" + adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/model" +) + +func init() { + if err := adp.RegisterFactory(model.RegistryTypeHelmHub, func(registry *model.Registry) (adp.Adapter, error) { + return newAdapter(registry) + }); err != nil { + log.Errorf("failed to register factory for %s: %v", model.RegistryTypeHelmHub, err) + return + } + log.Infof("the factory for adapter %s registered", model.RegistryTypeHelmHub) +} + +type adapter struct { + registry *model.Registry + client *Client +} + +func newAdapter(registry *model.Registry) (*adapter, error) { + return &adapter{ + registry: registry, + client: NewClient(registry), + }, nil +} + +func (a *adapter) Info() (*model.RegistryInfo, error) { + return &model.RegistryInfo{ + Type: model.RegistryTypeHelmHub, + SupportedResourceTypes: []model.ResourceType{ + model.ResourceTypeChart, + }, + SupportedResourceFilters: []*model.FilterStyle{ + { + Type: model.FilterTypeName, + Style: model.FilterStyleTypeText, + }, + { + Type: model.FilterTypeTag, + Style: model.FilterStyleTypeText, + }, + }, + SupportedTriggers: []model.TriggerType{ + model.TriggerTypeManual, + model.TriggerTypeScheduled, + }, + }, nil +} + +func (a *adapter) PrepareForPush(resources []*model.Resource) error { + return errors.New("not supported") +} + +// HealthCheck checks health status of a registry +func (a *adapter) HealthCheck() (model.HealthStatus, error) { + return model.Healthy, nil +} diff --git a/src/replication/adapter/helmhub/adapter_test.go b/src/replication/adapter/helmhub/adapter_test.go new file mode 100644 index 000000000..0b759038d --- /dev/null +++ b/src/replication/adapter/helmhub/adapter_test.go @@ -0,0 +1,44 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package helmhub + +import ( + "testing" + + "github.com/goharbor/harbor/src/replication/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInfo(t *testing.T) { + adapter := &adapter{} + info, err := adapter.Info() + require.Nil(t, err) + require.Equal(t, 1, len(info.SupportedResourceTypes)) + assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[0]) +} + +func TestPrepareForPush(t *testing.T) { + adapter := &adapter{} + err := adapter.PrepareForPush(nil) + require.NotNil(t, err) +} + +func TestHealthCheck(t *testing.T) { + adapter := &adapter{} + status, err := adapter.HealthCheck() + require.Equal(t, model.Healthy, string(status)) + require.Nil(t, err) +} diff --git a/src/replication/adapter/helmhub/chart.go b/src/replication/adapter/helmhub/chart.go new file mode 100644 index 000000000..5c46b9e64 --- /dev/null +++ b/src/replication/adapter/helmhub/chart.go @@ -0,0 +1,44 @@ +package helmhub + +type chart struct { + ID string `json:"id"` + Type string `json:"type"` +} + +type chartList struct { + Data []*chart `json:"data"` +} + +type chartAttributes struct { + Version string `json:"version"` + URLs []string `json:"urls"` +} + +type chartRepo struct { + Name string `json:"name"` + URL string `json:"url"` +} + +type chartData struct { + Name string `json:"name"` + Repo *chartRepo `json:"repo"` +} + +type chartInfo struct { + Data *chartData `json:"data"` +} + +type chartRelationships struct { + Chart *chartInfo `json:"chart"` +} + +type chartVersion struct { + ID string `json:"id"` + Type string `json:"type"` + Attributes *chartAttributes `json:"attributes"` + Relationships *chartRelationships `json:"relationships"` +} + +type chartVersionList struct { + Data []*chartVersion `json:"data"` +} diff --git a/src/replication/adapter/helmhub/chart_registry.go b/src/replication/adapter/helmhub/chart_registry.go new file mode 100644 index 000000000..76f25bae4 --- /dev/null +++ b/src/replication/adapter/helmhub/chart_registry.go @@ -0,0 +1,145 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package helmhub + +import ( + "fmt" + "io" + "net/http" + "strings" + + "github.com/goharbor/harbor/src/common/utils/log" + adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/model" + "github.com/pkg/errors" +) + +func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) { + charts, err := a.client.fetchCharts() + if err != nil { + return nil, err + } + + resources := []*model.Resource{} + repositories := []*adp.Repository{} + for _, chart := range charts.Data { + repository := &adp.Repository{ + ResourceType: string(model.ResourceTypeChart), + Name: chart.ID, + } + repositories = append(repositories, repository) + } + + for _, filter := range filters { + if err = filter.DoFilter(&repositories); err != nil { + return nil, err + } + } + + for _, repository := range repositories { + versionList, err := a.client.fetchChartDetail(repository.Name) + if err != nil { + log.Errorf("fetch chart detail: %v", err) + continue + } + + vTags := []*adp.VTag{} + for _, version := range versionList.Data { + vTags = append(vTags, &adp.VTag{ + Name: version.Attributes.Version, + ResourceType: string(model.ResourceTypeChart), + }) + } + + for _, filter := range filters { + if err = filter.DoFilter(&vTags); err != nil { + return nil, err + } + } + + for _, vTag := range vTags { + resources = append(resources, &model.Resource{ + Type: model.ResourceTypeChart, + Registry: a.registry, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: repository.Name, + }, + Vtags: []string{vTag.Name}, + }, + }) + } + } + return resources, nil +} + +func (a *adapter) ChartExist(name, version string) (bool, error) { + versionList, err := a.client.fetchChartDetail(name) + if err != nil && err == ErrHTTPNotFound { + return false, nil + } else if err != nil { + return false, err + } + + for _, v := range versionList.Data { + if v.Attributes.Version == version { + return true, nil + } + } + return false, nil +} + +func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) { + versionList, err := a.client.fetchChartDetail(name) + if err != nil { + return nil, err + } + + for _, v := range versionList.Data { + if v.Attributes.Version == version { + return a.download(v) + } + } + return nil, nil +} + +func (a *adapter) download(version *chartVersion) (io.ReadCloser, error) { + if version.Attributes.URLs == nil || len(version.Attributes.URLs) == 0 || len(version.Attributes.URLs[0]) == 0 { + return nil, fmt.Errorf("cannot got the download url for chart %s", version.ID) + } + + url := strings.ToLower(version.Attributes.URLs[0]) + if !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) { + url = fmt.Sprintf("%s/charts/%s", version.Relationships.Chart.Data.Repo.URL, url) + } + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := a.client.do(req) + if err != nil { + return nil, err + } + return resp.Body, nil +} + +func (a *adapter) UploadChart(name, version string, chart io.Reader) error { + return errors.New("not supported") +} + +func (a *adapter) DeleteChart(name, version string) error { + return errors.New("not supported") +} diff --git a/src/replication/adapter/helmhub/chart_registry_test.go b/src/replication/adapter/helmhub/chart_registry_test.go new file mode 100644 index 000000000..504d14f20 --- /dev/null +++ b/src/replication/adapter/helmhub/chart_registry_test.go @@ -0,0 +1,94 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package helmhub + +import ( + "testing" + + "github.com/goharbor/harbor/src/replication/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFetchCharts(t *testing.T) { + adapter, err := newAdapter(nil) + require.Nil(t, err) + // filter 1 + filters := []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "k*/*", + }, + } + resources, err := adapter.FetchCharts(filters) + require.Nil(t, err) + assert.NotZero(t, len(resources)) + assert.Equal(t, model.ResourceTypeChart, resources[0].Type) + assert.Equal(t, 1, len(resources[0].Metadata.Vtags)) + assert.NotNil(t, resources[0].Metadata.Vtags[0]) + // filter 2 + filters = []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "harbor/*", + }, + } + resources, err = adapter.FetchCharts(filters) + require.Nil(t, err) + assert.NotZero(t, len(resources)) + assert.Equal(t, model.ResourceTypeChart, resources[0].Type) + assert.Equal(t, "harbor/harbor", resources[0].Metadata.Repository.Name) + assert.Equal(t, 1, len(resources[0].Metadata.Vtags)) + assert.NotNil(t, resources[0].Metadata.Vtags[0]) +} + +func TestChartExist(t *testing.T) { + adapter, err := newAdapter(nil) + require.Nil(t, err) + exist, err := adapter.ChartExist("harbor/harbor", "1.0.0") + require.Nil(t, err) + require.True(t, exist) +} + +func TestChartExist2(t *testing.T) { + adapter, err := newAdapter(nil) + require.Nil(t, err) + exist, err := adapter.ChartExist("goharbor/harbor", "1.0.0") + require.Nil(t, err) + require.False(t, exist) + + exist, err = adapter.ChartExist("harbor/harbor", "1.0.100") + require.Nil(t, err) + require.False(t, exist) +} + +func TestDownloadChart(t *testing.T) { + adapter, err := newAdapter(nil) + require.Nil(t, err) + _, err = adapter.DownloadChart("harbor/harbor", "1.0.0") + require.Nil(t, err) +} + +func TestUploadChart(t *testing.T) { + adapter := &adapter{} + err := adapter.UploadChart("library/harbor", "1.0", nil) + require.NotNil(t, err) +} + +func TestDeleteChart(t *testing.T) { + adapter := &adapter{} + err := adapter.DeleteChart("library/harbor", "1.0") + require.NotNil(t, err) +} diff --git a/src/replication/adapter/helmhub/client.go b/src/replication/adapter/helmhub/client.go new file mode 100644 index 000000000..055e5c08b --- /dev/null +++ b/src/replication/adapter/helmhub/client.go @@ -0,0 +1,97 @@ +package helmhub + +import ( + "encoding/json" + "fmt" + "github.com/goharbor/harbor/src/replication/model" + "github.com/goharbor/harbor/src/replication/util" + "github.com/pkg/errors" + "io/ioutil" + "net/http" +) + +// ErrHTTPNotFound defines the return error when receiving 404 response code +var ErrHTTPNotFound = errors.New("Not Found") + +// Client is a client to interact with HelmHub +type Client struct { + client *http.Client +} + +// NewClient creates a new HelmHub client. +func NewClient(registry *model.Registry) *Client { + return &Client{ + client: &http.Client{ + Transport: util.GetHTTPTransport(false), + }, + } +} + +// fetchCharts fetches the chart list from helm hub. +func (c *Client) fetchCharts() (*chartList, error) { + request, err := http.NewRequest(http.MethodGet, baseURL+listCharts, nil) + if err != nil { + return nil, err + } + + resp, err := c.client.Do(request) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fetch chart list error %d: %s", resp.StatusCode, string(body)) + } + + list := &chartList{} + err = json.Unmarshal(body, list) + if err != nil { + return nil, fmt.Errorf("unmarshal chart list response error: %v", err) + } + + return list, nil +} + +// fetchChartDetail fetches the chart detail of a chart from helm hub. +func (c *Client) fetchChartDetail(chartName string) (*chartVersionList, error) { + request, err := http.NewRequest(http.MethodGet, baseURL+listVersions(chartName), nil) + if err != nil { + return nil, err + } + + resp, err := c.client.Do(request) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNotFound { + return nil, fmt.Errorf("fetch chart detail error %d: %s", resp.StatusCode, string(body)) + } else if resp.StatusCode == http.StatusNotFound { + return nil, ErrHTTPNotFound + } + + list := &chartVersionList{} + err = json.Unmarshal(body, list) + if err != nil { + return nil, fmt.Errorf("unmarshal chart detail response error: %v", err) + } + + return list, nil +} + +// do work as a proxy of Do function from net.http +func (c *Client) do(req *http.Request) (*http.Response, error) { + return c.client.Do(req) +} diff --git a/src/replication/adapter/helmhub/consts.go b/src/replication/adapter/helmhub/consts.go new file mode 100644 index 000000000..dab17bfcf --- /dev/null +++ b/src/replication/adapter/helmhub/consts.go @@ -0,0 +1,12 @@ +package helmhub + +import "fmt" + +const ( + baseURL = "https://hub.helm.sh" + listCharts = "/api/chartsvc/v1/charts" +) + +func listVersions(chartName string) string { + return fmt.Sprintf("/api/chartsvc/v1/charts/%s/versions", chartName) +} diff --git a/src/replication/model/registry.go b/src/replication/model/registry.go index 4f2459fe9..f5af7e7e1 100644 --- a/src/replication/model/registry.go +++ b/src/replication/model/registry.go @@ -30,6 +30,8 @@ const ( RegistryTypeAwsEcr RegistryType = "aws-ecr" RegistryTypeAzureAcr RegistryType = "azure-acr" + RegistryTypeHelmHub RegistryType = "helm-hub" + FilterStyleTypeText = "input" FilterStyleTypeRadio = "radio" FilterStyleTypeList = "list" From f32670058a542fb341d64989f91fea777916df50 Mon Sep 17 00:00:00 2001 From: mmpei Date: Wed, 10 Jul 2019 17:31:59 +0800 Subject: [PATCH 2/3] Fix by comments Signed-off-by: mmpei --- src/replication/adapter/helmhub/adapter.go | 6 +++++- .../adapter/helmhub/adapter_test.go | 2 +- .../adapter/helmhub/chart_registry.go | 4 ++-- src/replication/adapter/helmhub/client.go | 19 +++++++++++++++++++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/replication/adapter/helmhub/adapter.go b/src/replication/adapter/helmhub/adapter.go index 28c855dc8..45fb7a0a3 100644 --- a/src/replication/adapter/helmhub/adapter.go +++ b/src/replication/adapter/helmhub/adapter.go @@ -72,5 +72,9 @@ func (a *adapter) PrepareForPush(resources []*model.Resource) error { // HealthCheck checks health status of a registry func (a *adapter) HealthCheck() (model.HealthStatus, error) { - return model.Healthy, nil + err := a.client.checkHealthy() + if err == nil { + return model.Healthy, nil + } + return model.Unhealthy, err } diff --git a/src/replication/adapter/helmhub/adapter_test.go b/src/replication/adapter/helmhub/adapter_test.go index 0b759038d..ee22fc6dd 100644 --- a/src/replication/adapter/helmhub/adapter_test.go +++ b/src/replication/adapter/helmhub/adapter_test.go @@ -37,7 +37,7 @@ func TestPrepareForPush(t *testing.T) { } func TestHealthCheck(t *testing.T) { - adapter := &adapter{} + adapter, _ := newAdapter(nil) status, err := adapter.HealthCheck() require.Equal(t, model.Healthy, string(status)) require.Nil(t, err) diff --git a/src/replication/adapter/helmhub/chart_registry.go b/src/replication/adapter/helmhub/chart_registry.go index 76f25bae4..db4b79b74 100644 --- a/src/replication/adapter/helmhub/chart_registry.go +++ b/src/replication/adapter/helmhub/chart_registry.go @@ -52,7 +52,7 @@ func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error versionList, err := a.client.fetchChartDetail(repository.Name) if err != nil { log.Errorf("fetch chart detail: %v", err) - continue + return nil, err } vTags := []*adp.VTag{} @@ -112,7 +112,7 @@ func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) { return a.download(v) } } - return nil, nil + return nil, errors.New("chart not found") } func (a *adapter) download(version *chartVersion) (io.ReadCloser, error) { diff --git a/src/replication/adapter/helmhub/client.go b/src/replication/adapter/helmhub/client.go index 055e5c08b..a07ebb246 100644 --- a/src/replication/adapter/helmhub/client.go +++ b/src/replication/adapter/helmhub/client.go @@ -91,6 +91,25 @@ func (c *Client) fetchChartDetail(chartName string) (*chartVersionList, error) { return list, nil } +func (c *Client) checkHealthy() error { + request, err := http.NewRequest(http.MethodGet, baseURL, nil) + if err != nil { + return err + } + + resp, err := c.client.Do(request) + if err != nil { + return err + } + defer resp.Body.Close() + + ioutil.ReadAll(resp.Body) + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return nil + } + return errors.New("helm hub is unhealthy") +} + // do work as a proxy of Do function from net.http func (c *Client) do(req *http.Request) (*http.Response, error) { return c.client.Do(req) From 9f777ed43f0062f5c9f3d1881e5c059fa0e642da Mon Sep 17 00:00:00 2001 From: peimingming Date: Fri, 19 Jul 2019 17:04:40 +0800 Subject: [PATCH 3/3] Fix by comments Signed-off-by: peimingming --- src/replication/adapter/helmhub/chart_registry.go | 9 +++++---- src/replication/adapter/helmhub/client.go | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/replication/adapter/helmhub/chart_registry.go b/src/replication/adapter/helmhub/chart_registry.go index db4b79b74..daba32952 100644 --- a/src/replication/adapter/helmhub/chart_registry.go +++ b/src/replication/adapter/helmhub/chart_registry.go @@ -87,9 +87,10 @@ func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error func (a *adapter) ChartExist(name, version string) (bool, error) { versionList, err := a.client.fetchChartDetail(name) - if err != nil && err == ErrHTTPNotFound { - return false, nil - } else if err != nil { + if err != nil { + if err == ErrHTTPNotFound { + return false, nil + } return false, err } @@ -116,7 +117,7 @@ func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) { } func (a *adapter) download(version *chartVersion) (io.ReadCloser, error) { - if version.Attributes.URLs == nil || len(version.Attributes.URLs) == 0 || len(version.Attributes.URLs[0]) == 0 { + if len(version.Attributes.URLs) == 0 || len(version.Attributes.URLs[0]) == 0 { return nil, fmt.Errorf("cannot got the download url for chart %s", version.ID) } diff --git a/src/replication/adapter/helmhub/client.go b/src/replication/adapter/helmhub/client.go index a07ebb246..c69b0d7a7 100644 --- a/src/replication/adapter/helmhub/client.go +++ b/src/replication/adapter/helmhub/client.go @@ -76,10 +76,10 @@ func (c *Client) fetchChartDetail(chartName string) (*chartVersionList, error) { return nil, err } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNotFound { - return nil, fmt.Errorf("fetch chart detail error %d: %s", resp.StatusCode, string(body)) - } else if resp.StatusCode == http.StatusNotFound { + if resp.StatusCode == http.StatusNotFound { return nil, ErrHTTPNotFound + } else if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fetch chart detail error %d: %s", resp.StatusCode, string(body)) } list := &chartVersionList{}