fix(replication): refactor quay adapter to fix authorization and support quay.io and enterprise quay (#10317)

Signed-off-by: chlins <chlins.zhang@gmail.com>

make/migrations/postgresql/0040_2.1.0_schema.up.sql

@@ -111,4 +111,7 @@ BEGIN
 END $$;
 
 ALTER TABLE schedule DROP COLUMN IF EXISTS job_id;
-ALTER TABLE schedule DROP COLUMN IF EXISTS status;
+ALTER TABLE schedule DROP COLUMN IF EXISTS status;
+
+/*replication quay.io update vendor type*/
+UPDATE registry SET type = 'quay' WHERE type = 'quay-io';

src/jobservice/job/impl/replication/replication.go

@@ -45,7 +45,7 @@ import (
 	// register the Jfrog Artifactory adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/jfrog"
 	// register the Quay.io adapter
-	_ "github.com/goharbor/harbor/src/replication/adapter/quayio"
+	_ "github.com/goharbor/harbor/src/replication/adapter/quay"
 	// register the Helm Hub adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/helmhub"
 	// register the GitLab adapter

src/replication/adapter/quay/adapter.go

@@ -1,14 +1,17 @@
-package quayio
+package quay
 
 import (
 	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"strings"
 
+	"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
+
 	common_http "github.com/goharbor/harbor/src/common/http"
 	"github.com/goharbor/harbor/src/common/http/modifier"
 	"github.com/goharbor/harbor/src/lib/log"
@@ -25,31 +28,51 @@ var (
 
 type adapter struct {
 	*native.Adapter
-	registry *model.Registry
+	autoCreateNs bool
-	client   *common_http.Client
+	registry     *model.Registry
+	client       *common_http.Client
 }
 
 func init() {
-	err := adp.RegisterFactory(model.RegistryTypeQuayio, new(factory))
+	err := adp.RegisterFactory(model.RegistryTypeQuay, new(factory))
 	if err != nil {
-		log.Errorf("failed to register factory for Quay.io: %v", err)
+		log.Errorf("failed to register factory for Quay: %v", err)
 		return
 	}
-	log.Infof("the factory of Quay.io adapter was registered")
+	log.Infof("the factory of Quay adapter was registered")
 }
 
 func newAdapter(registry *model.Registry) (*adapter, error) {
-	modifiers := []modifier.Modifier{}
+	var modifiers []modifier.Modifier
-	var authorizer modifier.Modifier
+
-	if registry.Credential != nil && len(registry.Credential.AccessKey) != 0 {
+	var (
-		authorizer = NewAPIKeyAuthorizer("Authorization", fmt.Sprintf("Bearer %s", registry.Credential.AccessKey), APIKeyInHeader)
+		autoCreateNs                      bool
+		basicAuthorizer, apiKeyAuthorizer modifier.Modifier
+	)
+
+	if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 {
+		var jsonCred cred
+		err := json.Unmarshal([]byte(registry.Credential.AccessSecret), &jsonCred)
+		if err != nil {
+			return nil, err
+		}
+		basicAuthorizer = basic.NewAuthorizer(jsonCred.AccountName, jsonCred.DockerCliPassword)
+		if len(jsonCred.OAuth2Token) != 0 {
+			autoCreateNs = true
+			apiKeyAuthorizer = NewAPIKeyAuthorizer("Authorization", fmt.Sprintf("Bearer %s", jsonCred.OAuth2Token), APIKeyInHeader)
+		}
 	}
-	if authorizer != nil {
+
-		modifiers = append(modifiers, authorizer)
+	nativeRegistryAdapter := native.NewAdapterWithAuthorizer(registry, basicAuthorizer)
+
+	if apiKeyAuthorizer != nil {
+		modifiers = append(modifiers, apiKeyAuthorizer)
 	}
+
 	return &adapter{
-		Adapter:  native.NewAdapterWithAuthorizer(registry, authorizer),
+		Adapter:      nativeRegistryAdapter,
-		registry: registry,
+		autoCreateNs: autoCreateNs,
+		registry:     registry,
 		client: common_http.NewClient(
 			&http.Client{
 				Transport: util.GetHTTPTransport(registry.Insecure),
@@ -69,13 +92,22 @@ func (f *factory) Create(r *model.Registry) (adp.Adapter, error) {
 
 // AdapterPattern ...
 func (f *factory) AdapterPattern() *model.AdapterPattern {
-	return nil
+	info := &model.AdapterPattern{
+		EndpointPattern: model.NewDefaultEndpointPattern(),
+		CredentialPattern: &model.CredentialPattern{
+			AccessKeyType:    model.AccessKeyTypeFix,
+			AccessKeyData:    "json_file",
+			AccessSecretType: model.AccessSecretTypeFile,
+			AccessSecretData: "",
+		},
+	}
+	return info
 }
 
 // Info returns information of the registry
 func (a *adapter) Info() (*model.RegistryInfo, error) {
 	return &model.RegistryInfo{
-		Type: model.RegistryTypeQuayio,
+		Type: model.RegistryTypeQuay,
 		SupportedResourceTypes: []model.ResourceType{
 			model.ResourceTypeImage,
 		},
@@ -109,6 +141,9 @@ func (a *adapter) HealthCheck() (model.HealthStatus, error) {
 // PrepareForPush does the prepare work that needed for pushing/uploading the resource
 // eg: create the namespace or repository
 func (a *adapter) PrepareForPush(resources []*model.Resource) error {
+	if !a.autoCreateNs {
+		return nil
+	}
 	namespaces := []string{}
 	for _, resource := range resources {
 		if resource == nil {
@@ -133,14 +168,14 @@ func (a *adapter) PrepareForPush(resources []*model.Resource) error {
 			Name: namespace,
 		})
 		if err != nil {
-			return fmt.Errorf("create namespace '%s' in Quay.io error: %v", namespace, err)
+			return fmt.Errorf("create namespace '%s' in Quay error: %v", namespace, err)
 		}
 		log.Debugf("namespace %s created", namespace)
 	}
 	return nil
 }
 
-// createNamespace creates a new namespace in Quay.io
+// createNamespace creates a new namespace in Quay
 func (a *adapter) createNamespace(namespace *model.Namespace) error {
 	ns, err := a.getNamespace(namespace.Name)
 	if err != nil {
@@ -149,24 +184,25 @@ func (a *adapter) createNamespace(namespace *model.Namespace) error {
 
 	// If the namespace already exist, return succeeded directly.
 	if ns != nil {
-		log.Infof("Namespace %s already exist in Quay.io, skip it.", namespace.Name)
+		log.Infof("Namespace %s already exist in Quay, skip it.", namespace.Name)
 		return nil
 	}
 
 	org := &orgCreate{
 		Name:  namespace.Name,
-		Email: namespace.GetStringMetadata("email", namespace.Name),
+		Email: fmt.Sprintf("%s@quay.io", namespace.Name),
 	}
 	b, err := json.Marshal(org)
 	if err != nil {
 		return err
 	}
 
-	req, err := http.NewRequest(http.MethodPost, buildOrgURL(""), bytes.NewReader(b))
+	req, err := http.NewRequest(http.MethodPost, buildOrgURL(a.registry.URL, ""), bytes.NewBuffer(b))
 	if err != nil {
 		return err
 	}
 
+	req.Header.Set("Content-Type", "application/json")
 	resp, err := a.client.Do(req)
 	if err != nil {
 		return err
@@ -185,9 +221,9 @@ func (a *adapter) createNamespace(namespace *model.Namespace) error {
 	return fmt.Errorf("%d -- %s", resp.StatusCode, body)
 }
 
-// getNamespace get namespace from Quay.io, if the namespace not found, two nil would be returned.
+// getNamespace get namespace from Quay, if the namespace not found, two nil would be returned.
 func (a *adapter) getNamespace(namespace string) (*model.Namespace, error) {
-	req, err := http.NewRequest(http.MethodGet, buildOrgURL(namespace), nil)
+	req, err := http.NewRequest(http.MethodGet, buildOrgURL(a.registry.URL, namespace), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -216,3 +252,35 @@ func (a *adapter) getNamespace(namespace string) (*model.Namespace, error) {
 		Name: namespace,
 	}, nil
 }
+
+// PushManifest ...
+func (a *adapter) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) {
+	digest, err := a.Adapter.PushManifest(repository, reference, mediaType, payload)
+	if err != nil {
+		if comErr, ok := err.(*common_http.Error); ok {
+			if comErr.Code == http.StatusAccepted {
+				return digest, nil
+			}
+		}
+	}
+	return digest, err
+}
+
+// PullBlob ...
+func (a *adapter) PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error) {
+	size, blob, err = a.Adapter.PullBlob(repository, digest)
+	if err != nil && blob != nil {
+		if size == 0 {
+			var data []byte
+			defer blob.Close()
+			data, err = ioutil.ReadAll(blob)
+			if err != nil {
+				return
+			}
+			size = int64(len(data))
+			blob = ioutil.NopCloser(bytes.NewReader(data))
+			return size, blob, nil
+		}
+	}
+	return
+}

src/replication/adapter/quay/adapter_test.go

@@ -1,4 +1,4 @@
-package quayio
+package quay
 
 import (
 	"testing"
@@ -9,9 +9,9 @@ import (
 )
 
 func getMockAdapter(t *testing.T) adp.Adapter {
-	factory, _ := adp.GetFactory(model.RegistryTypeQuayio)
+	factory, _ := adp.GetFactory(model.RegistryTypeQuay)
 	adapter, err := factory.Create(&model.Registry{
-		Type: model.RegistryTypeQuayio,
+		Type: model.RegistryTypeQuay,
 		URL:  "https://quay.io",
 	})
 	assert.Nil(t, err)
@@ -22,7 +22,7 @@ func TestAdapter_NewAdapter(t *testing.T) {
 	assert.Nil(t, factory)
 	assert.NotNil(t, err)
 
-	factory, err = adp.GetFactory(model.RegistryTypeQuayio)
+	factory, err = adp.GetFactory(model.RegistryTypeQuay)
 	assert.Nil(t, err)
 	assert.NotNil(t, factory)
 }

src/replication/adapter/quay/apikey.go

@@ -1,4 +1,4 @@
-package quayio
+package quay
 
 import (
 	"fmt"

src/replication/adapter/quay/apikey_test.go

@@ -1,4 +1,4 @@
-package quayio
+package quay
 
 import (
 	"net/http"

src/replication/adapter/quay/types.go

@@ -0,0 +1,21 @@
+package quay
+
+import (
+	"fmt"
+	"strings"
+)
+
+type cred struct {
+	OAuth2Token       string `json:"oauth2_token"`
+	AccountName       string `json:"account_name"`
+	DockerCliPassword string `json:"docker_cli_password"`
+}
+
+type orgCreate struct {
+	Name  string `json:"name"`
+	Email string `json:"email"`
+}
+
+func buildOrgURL(endpoint, orgName string) string {
+	return fmt.Sprintf("%s/api/v1/organization/%s", strings.TrimRight(endpoint, "/"), orgName)
+}

src/replication/adapter/quayio/types.go

@@ -1,12 +0,0 @@
-package quayio
-
-import "fmt"
-
-type orgCreate struct {
-	Name  string `json:"name"`
-	Email string `json:"email"`
-}
-
-func buildOrgURL(orgName string) string {
-	return fmt.Sprintf("https://quay.io/api/v1/organization/%s", orgName)
-}

src/replication/model/registry.go

@@ -29,7 +29,7 @@ const (
 	RegistryTypeAzureAcr         RegistryType = "azure-acr"
 	RegistryTypeAliAcr           RegistryType = "ali-acr"
 	RegistryTypeJfrogArtifactory RegistryType = "jfrog-artifactory"
-	RegistryTypeQuayio           RegistryType = "quay-io"
+	RegistryTypeQuay             RegistryType = "quay"
 	RegistryTypeGitLab           RegistryType = "gitlab"
 
 	RegistryTypeHelmHub RegistryType = "helm-hub"

src/replication/replication.go

@@ -46,7 +46,7 @@ import (
 	// register the Jfrog Artifactory adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/jfrog"
 	// register the Quay.io adapter
-	_ "github.com/goharbor/harbor/src/replication/adapter/quayio"
+	_ "github.com/goharbor/harbor/src/replication/adapter/quay"
 	// register the Helm Hub adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/helmhub"
 	// register the GitLab adapter