From 4ab3b864ae62f7270b8f9d5d290966b851ef2716 Mon Sep 17 00:00:00 2001 From: chlins Date: Mon, 5 Aug 2019 13:28:39 +0800 Subject: [PATCH] feat: add image replication adapter for quay.io Signed-off-by: chlins --- src/common/utils/registry/auth/apikey.go | 45 ++++ src/common/utils/registry/auth/apikey_test.go | 50 ++++ .../job/impl/replication/replication.go | 2 + src/replication/adapter/quayio/adapter.go | 214 ++++++++++++++++++ .../adapter/quayio/adapter_test.go | 48 ++++ src/replication/adapter/quayio/types.go | 12 + src/replication/model/registry.go | 1 + src/replication/replication.go | 2 + 8 files changed, 374 insertions(+) create mode 100644 src/common/utils/registry/auth/apikey.go create mode 100644 src/common/utils/registry/auth/apikey_test.go create mode 100644 src/replication/adapter/quayio/adapter.go create mode 100644 src/replication/adapter/quayio/adapter_test.go create mode 100644 src/replication/adapter/quayio/types.go diff --git a/src/common/utils/registry/auth/apikey.go b/src/common/utils/registry/auth/apikey.go new file mode 100644 index 000000000..1dd02b16e --- /dev/null +++ b/src/common/utils/registry/auth/apikey.go @@ -0,0 +1,45 @@ +package auth + +import ( + "fmt" + "net/http" + + "github.com/goharbor/harbor/src/common/http/modifier" +) + +type apiKeyType = string + +const ( + // APIKeyInHeader sets auth content in header + APIKeyInHeader apiKeyType = "header" + // APIKeyInQuery sets auth content in url query + APIKeyInQuery apiKeyType = "query" +) + +type apiKeyAuthorizer struct { + key, value, in apiKeyType +} + +// NewAPIKeyAuthorizer returns a apikey authorizer +func NewAPIKeyAuthorizer(key, value, in apiKeyType) modifier.Modifier { + return &apiKeyAuthorizer{ + key: key, + value: value, + in: in, + } +} + +// Modify implements modifier.Modifier +func (a *apiKeyAuthorizer) Modify(r *http.Request) error { + switch a.in { + case APIKeyInHeader: + r.Header.Set(a.key, a.value) + return nil + case APIKeyInQuery: + query := r.URL.Query() + query.Add(a.key, a.value) + r.URL.RawQuery = query.Encode() + return nil + } + return fmt.Errorf("set api key in %s is invalid", a.in) +} diff --git a/src/common/utils/registry/auth/apikey_test.go b/src/common/utils/registry/auth/apikey_test.go new file mode 100644 index 000000000..ff6ef4133 --- /dev/null +++ b/src/common/utils/registry/auth/apikey_test.go @@ -0,0 +1,50 @@ +package auth + +import ( + "net/http" + "testing" + + "github.com/goharbor/harbor/src/common/http/modifier" + "github.com/stretchr/testify/assert" +) + +func TestAPIKeyAuthorizer(t *testing.T) { + type suite struct { + key string + value string + in string + } + + var ( + s suite + authorizer modifier.Modifier + request *http.Request + err error + ) + + // set in header + s = suite{key: "Authorization", value: "Basic abc", in: "header"} + authorizer = NewAPIKeyAuthorizer(s.key, s.value, s.in) + request, err = http.NewRequest(http.MethodGet, "http://example.com", nil) + assert.Nil(t, err) + err = authorizer.Modify(request) + assert.Nil(t, err) + assert.Equal(t, s.value, request.Header.Get(s.key)) + + // set in query + s = suite{key: "private_token", value: "abc", in: "query"} + authorizer = NewAPIKeyAuthorizer(s.key, s.value, s.in) + request, err = http.NewRequest(http.MethodGet, "http://example.com", nil) + assert.Nil(t, err) + err = authorizer.Modify(request) + assert.Nil(t, err) + assert.Equal(t, s.value, request.URL.Query().Get(s.key)) + + // set in invalid location + s = suite{key: "", value: "", in: "invalid"} + authorizer = NewAPIKeyAuthorizer(s.key, s.value, s.in) + request, err = http.NewRequest(http.MethodGet, "http://example.com", nil) + assert.Nil(t, err) + err = authorizer.Modify(request) + assert.NotNil(t, err) +} diff --git a/src/jobservice/job/impl/replication/replication.go b/src/jobservice/job/impl/replication/replication.go index d7a35cdbf..00e5bd72b 100644 --- a/src/jobservice/job/impl/replication/replication.go +++ b/src/jobservice/job/impl/replication/replication.go @@ -44,6 +44,8 @@ import ( _ "github.com/goharbor/harbor/src/replication/adapter/aliacr" // register the Jfrog Artifactory adapter _ "github.com/goharbor/harbor/src/replication/adapter/jfrog" + // register the Quay.io adapter + _ "github.com/goharbor/harbor/src/replication/adapter/quayio" // register the Helm Hub adapter _ "github.com/goharbor/harbor/src/replication/adapter/helmhub" ) diff --git a/src/replication/adapter/quayio/adapter.go b/src/replication/adapter/quayio/adapter.go new file mode 100644 index 000000000..01dc5c5d8 --- /dev/null +++ b/src/replication/adapter/quayio/adapter.go @@ -0,0 +1,214 @@ +package quayio + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "strings" + + common_http "github.com/goharbor/harbor/src/common/http" + "github.com/goharbor/harbor/src/common/http/modifier" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/common/utils/registry/auth" + adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/adapter/native" + "github.com/goharbor/harbor/src/replication/model" + "github.com/goharbor/harbor/src/replication/util" +) + +type adapter struct { + *native.Adapter + registry *model.Registry + client *common_http.Client +} + +func init() { + err := adp.RegisterFactory(model.RegistryTypeQuayio, func(registry *model.Registry) (adp.Adapter, error) { + return newAdapter(registry) + }) + if err != nil { + log.Errorf("failed to register factory for Quay.io: %v", err) + return + } + log.Infof("the factory of Quay.io adapter was registered") +} + +func newAdapter(registry *model.Registry) (*adapter, error) { + modifiers := []modifier.Modifier{ + &auth.UserAgentModifier{ + UserAgent: adp.UserAgentReplication, + }, + } + + var authorizer modifier.Modifier + if registry.Credential != nil && len(registry.Credential.AccessKey) != 0 { + authorizer = auth.NewAPIKeyAuthorizer("Authorization", fmt.Sprintf("Bearer %s", registry.Credential.AccessKey), auth.APIKeyInHeader) + } + + if authorizer != nil { + modifiers = append(modifiers, authorizer) + } + nativeRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer) + if err != nil { + return nil, err + } + + return &adapter{ + Adapter: nativeRegistryAdapter, + registry: registry, + client: common_http.NewClient( + &http.Client{ + Transport: util.GetHTTPTransport(registry.Insecure), + }, + modifiers..., + ), + }, nil +} + +// Info returns information of the registry +func (a *adapter) Info() (*model.RegistryInfo, error) { + return &model.RegistryInfo{ + Type: model.RegistryTypeQuayio, + SupportedResourceTypes: []model.ResourceType{ + model.ResourceTypeImage, + }, + SupportedResourceFilters: []*model.FilterStyle{ + { + Type: model.FilterTypeName, + Style: model.FilterStyleTypeText, + }, + { + Type: model.FilterTypeTag, + Style: model.FilterStyleTypeText, + }, + }, + SupportedTriggers: []model.TriggerType{ + model.TriggerTypeManual, + model.TriggerTypeScheduled, + }, + }, nil +} + +// HealthCheck checks health status of a registry +func (a *adapter) HealthCheck() (model.HealthStatus, error) { + err := a.PingSimple() + if err != nil { + return model.Unhealthy, nil + } + + return model.Healthy, nil +} + +// PrepareForPush does the prepare work that needed for pushing/uploading the resource +// eg: create the namespace or repository +func (a *adapter) PrepareForPush(resources []*model.Resource) error { + namespaces := []string{} + for _, resource := range resources { + if resource == nil { + return errors.New("the resource cannot be null") + } + if resource.Metadata == nil { + return errors.New("the metadata of resource cannot be null") + } + if resource.Metadata.Repository == nil { + return errors.New("the namespace of resource cannot be null") + } + if len(resource.Metadata.Repository.Name) == 0 { + return errors.New("the name of the namespace cannot be null") + } + paths := strings.Split(resource.Metadata.Repository.Name, "/") + namespace := paths[0] + namespaces = append(namespaces, namespace) + } + + for _, namespace := range namespaces { + err := a.createNamespace(&model.Namespace{ + Name: namespace, + }) + if err != nil { + return fmt.Errorf("create namespace '%s' in Quay.io error: %v", namespace, err) + } + log.Debugf("namespace %s created", namespace) + } + return nil +} + +// createNamespace creates a new namespace in Quay.io +func (a *adapter) createNamespace(namespace *model.Namespace) error { + ns, err := a.getNamespace(namespace.Name) + if err != nil { + return fmt.Errorf("check existence of namespace '%s' error: %v", namespace.Name, err) + } + + // If the namespace already exist, return succeeded directly. + if ns != nil { + log.Infof("Namespace %s already exist in Quay.io, skip it.", namespace.Name) + return nil + } + + org := &orgCreate{ + Name: namespace.Name, + Email: namespace.GetStringMetadata("email", namespace.Name), + } + b, err := json.Marshal(org) + if err != nil { + return err + } + + req, err := http.NewRequest(http.MethodPost, buildOrgURL(""), bytes.NewReader(b)) + if err != nil { + return err + } + + resp, err := a.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode/100 == 2 { + return nil + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + log.Errorf("create namespace error: %d -- %s", resp.StatusCode, string(body)) + return fmt.Errorf("%d -- %s", resp.StatusCode, body) +} + +// getNamespace get namespace from Quay.io, if the namespace not found, two nil would be returned. +func (a *adapter) getNamespace(namespace string) (*model.Namespace, error) { + req, err := http.NewRequest(http.MethodGet, buildOrgURL(namespace), nil) + if err != nil { + return nil, err + } + + resp, err := a.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } + + if resp.StatusCode/100 != 2 { + log.Errorf("get namespace error: %d -- %s", resp.StatusCode, string(body)) + return nil, fmt.Errorf("%d -- %s", resp.StatusCode, body) + } + + return &model.Namespace{ + Name: namespace, + }, nil +} diff --git a/src/replication/adapter/quayio/adapter_test.go b/src/replication/adapter/quayio/adapter_test.go new file mode 100644 index 000000000..a77218c0b --- /dev/null +++ b/src/replication/adapter/quayio/adapter_test.go @@ -0,0 +1,48 @@ +package quayio + +import ( + "testing" + + adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/model" + "github.com/stretchr/testify/assert" +) + +func getMockAdapter(t *testing.T) adp.Adapter { + factory, _ := adp.GetFactory(model.RegistryTypeQuayio) + adapter, err := factory(&model.Registry{ + Type: model.RegistryTypeQuayio, + URL: "https://quay.io", + }) + assert.Nil(t, err) + return adapter +} +func TestAdapter_NewAdapter(t *testing.T) { + factory, err := adp.GetFactory("BadName") + assert.Nil(t, factory) + assert.NotNil(t, err) + + factory, err = adp.GetFactory(model.RegistryTypeQuayio) + assert.Nil(t, err) + assert.NotNil(t, factory) +} + +func TestAdapter_HealthCheck(t *testing.T) { + health, err := getMockAdapter(t).HealthCheck() + assert.Nil(t, err) + assert.Equal(t, string(health), model.Healthy) +} + +func TestAdapter_Info(t *testing.T) { + info, err := getMockAdapter(t).Info() + assert.Nil(t, err) + t.Log(info) +} + +func TestAdapter_PullManifests(t *testing.T) { + quayAdapter := getMockAdapter(t) + registry, _, err := quayAdapter.(*adapter).PullManifest("quay/busybox", "latest", []string{}) + assert.Nil(t, err) + assert.NotNil(t, registry) + t.Log(registry) +} diff --git a/src/replication/adapter/quayio/types.go b/src/replication/adapter/quayio/types.go new file mode 100644 index 000000000..393dad058 --- /dev/null +++ b/src/replication/adapter/quayio/types.go @@ -0,0 +1,12 @@ +package quayio + +import "fmt" + +type orgCreate struct { + Name string `json:"name"` + Email string `json:"email"` +} + +func buildOrgURL(orgName string) string { + return fmt.Sprintf("https://quay.io/api/v1/organization/%s", orgName) +} diff --git a/src/replication/model/registry.go b/src/replication/model/registry.go index a31f50896..3fa35c5aa 100644 --- a/src/replication/model/registry.go +++ b/src/replication/model/registry.go @@ -31,6 +31,7 @@ const ( RegistryTypeAzureAcr RegistryType = "azure-acr" RegistryTypeAliAcr RegistryType = "ali-acr" RegistryTypeJfrogArtifactory RegistryType = "jfrog-artifactory" + RegistryTypeQuayio RegistryType = "quay-io" RegistryTypeHelmHub RegistryType = "helm-hub" diff --git a/src/replication/replication.go b/src/replication/replication.go index b06404874..4eb0dde29 100644 --- a/src/replication/replication.go +++ b/src/replication/replication.go @@ -45,6 +45,8 @@ import ( _ "github.com/goharbor/harbor/src/replication/adapter/aliacr" // register the Jfrog Artifactory adapter _ "github.com/goharbor/harbor/src/replication/adapter/jfrog" + // register the Quay.io adapter + _ "github.com/goharbor/harbor/src/replication/adapter/quayio" // register the Helm Hub adapter _ "github.com/goharbor/harbor/src/replication/adapter/helmhub" )