Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-25 03:35:21 +01:00
Code Issues Projects Releases Wiki Activity

fix(replication): refactor quay adapter to fix authorization and support quay.io and enterprise quay (#10317)

Browse Source 
Signed-off-by: chlins <chlins.zhang@gmail.com>
This commit is contained in:
chlins 2020-01-13 15:15:02 +08:00
parent 6b9929d848
commit b765cfe0ce
10 changed files with 125 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
make/migrations/postgresql/0040_2.1.0_schema.up.sql
View File

@ -111,4 +111,7 @@ BEGIN
END $$;
ALTER TABLE schedule DROP COLUMN IF EXISTS job_id;
ALTER TABLE schedule DROP COLUMN IF EXISTS status;
ALTER TABLE schedule DROP COLUMN IF EXISTS status;
/*replication quay.io update vendor type*/
UPDATE registry SET type = 'quay' WHERE type = 'quay-io';

2
src/jobservice/job/impl/replication/replication.go
View File

@ -45,7 +45,7 @@ import (
// register the Jfrog Artifactory adapter
_ "github.com/goharbor/harbor/src/replication/adapter/jfrog"
// register the Quay.io adapter
_ "github.com/goharbor/harbor/src/replication/adapter/quayio"
_ "github.com/goharbor/harbor/src/replication/adapter/quay"
// register the Helm Hub adapter
_ "github.com/goharbor/harbor/src/replication/adapter/helmhub"
// register the GitLab adapter

114
src/replication/adapter/quayio/adapter.go → src/replication/adapter/quay/adapter.go
View File

@ -1,14 +1,17 @@
package quayio
package quay
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"strings"
"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/lib/log"
@ -25,31 +28,51 @@ var (
type adapter struct {
*native.Adapter
registry *model.Registry
client *common_http.Client
autoCreateNs bool
registry *model.Registry
client *common_http.Client
}
func init() {
err := adp.RegisterFactory(model.RegistryTypeQuayio, new(factory))
err := adp.RegisterFactory(model.RegistryTypeQuay, new(factory))
if err != nil {
log.Errorf("failed to register factory for Quay.io: %v", err)
log.Errorf("failed to register factory for Quay: %v", err)
return
}
log.Infof("the factory of Quay.io adapter was registered")
log.Infof("the factory of Quay adapter was registered")
}
func newAdapter(registry *model.Registry) (*adapter, error) {
modifiers := []modifier.Modifier{}
var authorizer modifier.Modifier
if registry.Credential != nil && len(registry.Credential.AccessKey) != 0 {
authorizer = NewAPIKeyAuthorizer("Authorization", fmt.Sprintf("Bearer %s", registry.Credential.AccessKey), APIKeyInHeader)
var modifiers []modifier.Modifier
var (
autoCreateNs bool
basicAuthorizer, apiKeyAuthorizer modifier.Modifier
)
if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 {
var jsonCred cred
err := json.Unmarshal([]byte(registry.Credential.AccessSecret), &jsonCred)
if err != nil {
return nil, err
}
basicAuthorizer = basic.NewAuthorizer(jsonCred.AccountName, jsonCred.DockerCliPassword)
if len(jsonCred.OAuth2Token) != 0 {
autoCreateNs = true
apiKeyAuthorizer = NewAPIKeyAuthorizer("Authorization", fmt.Sprintf("Bearer %s", jsonCred.OAuth2Token), APIKeyInHeader)
}
}
if authorizer != nil {
modifiers = append(modifiers, authorizer)
nativeRegistryAdapter := native.NewAdapterWithAuthorizer(registry, basicAuthorizer)
if apiKeyAuthorizer != nil {
modifiers = append(modifiers, apiKeyAuthorizer)
}
return &adapter{
Adapter: native.NewAdapterWithAuthorizer(registry, authorizer),
registry: registry,
Adapter: nativeRegistryAdapter,
autoCreateNs: autoCreateNs,
registry: registry,
client: common_http.NewClient(
&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
@ -69,13 +92,22 @@ func (f *factory) Create(r *model.Registry) (adp.Adapter, error) {
// AdapterPattern ...
func (f *factory) AdapterPattern() *model.AdapterPattern {
return nil
info := &model.AdapterPattern{
EndpointPattern: model.NewDefaultEndpointPattern(),
CredentialPattern: &model.CredentialPattern{
AccessKeyType: model.AccessKeyTypeFix,
AccessKeyData: "json_file",
AccessSecretType: model.AccessSecretTypeFile,
AccessSecretData: "",
},
}
return info
}
// Info returns information of the registry
func (a *adapter) Info() (*model.RegistryInfo, error) {
return &model.RegistryInfo{
Type: model.RegistryTypeQuayio,
Type: model.RegistryTypeQuay,
SupportedResourceTypes: []model.ResourceType{
model.ResourceTypeImage,
},
@ -109,6 +141,9 @@ func (a *adapter) HealthCheck() (model.HealthStatus, error) {
// PrepareForPush does the prepare work that needed for pushing/uploading the resource
// eg: create the namespace or repository
func (a *adapter) PrepareForPush(resources []*model.Resource) error {
if !a.autoCreateNs {
return nil
}
namespaces := []string{}
for _, resource := range resources {
if resource == nil {
@ -133,14 +168,14 @@ func (a *adapter) PrepareForPush(resources []*model.Resource) error {
Name: namespace,
})
if err != nil {
return fmt.Errorf("create namespace '%s' in Quay.io error: %v", namespace, err)
return fmt.Errorf("create namespace '%s' in Quay error: %v", namespace, err)
}
log.Debugf("namespace %s created", namespace)
}
return nil
}
// createNamespace creates a new namespace in Quay.io
// createNamespace creates a new namespace in Quay
func (a *adapter) createNamespace(namespace *model.Namespace) error {
ns, err := a.getNamespace(namespace.Name)
if err != nil {
@ -149,24 +184,25 @@ func (a *adapter) createNamespace(namespace *model.Namespace) error {
// If the namespace already exist, return succeeded directly.
if ns != nil {
log.Infof("Namespace %s already exist in Quay.io, skip it.", namespace.Name)
log.Infof("Namespace %s already exist in Quay, skip it.", namespace.Name)
return nil
}
org := &orgCreate{
Name: namespace.Name,
Email: namespace.GetStringMetadata("email", namespace.Name),
Email: fmt.Sprintf("%s@quay.io", namespace.Name),
}
b, err := json.Marshal(org)
if err != nil {
return err
}
req, err := http.NewRequest(http.MethodPost, buildOrgURL(""), bytes.NewReader(b))
req, err := http.NewRequest(http.MethodPost, buildOrgURL(a.registry.URL, ""), bytes.NewBuffer(b))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := a.client.Do(req)
if err != nil {
return err
@ -185,9 +221,9 @@ func (a *adapter) createNamespace(namespace *model.Namespace) error {
return fmt.Errorf("%d -- %s", resp.StatusCode, body)
}
// getNamespace get namespace from Quay.io, if the namespace not found, two nil would be returned.
// getNamespace get namespace from Quay, if the namespace not found, two nil would be returned.
func (a *adapter) getNamespace(namespace string) (*model.Namespace, error) {
req, err := http.NewRequest(http.MethodGet, buildOrgURL(namespace), nil)
req, err := http.NewRequest(http.MethodGet, buildOrgURL(a.registry.URL, namespace), nil)
if err != nil {
return nil, err
}
@ -216,3 +252,35 @@ func (a *adapter) getNamespace(namespace string) (*model.Namespace, error) {
Name: namespace,
}, nil
}
// PushManifest ...
func (a *adapter) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) {
digest, err := a.Adapter.PushManifest(repository, reference, mediaType, payload)
if err != nil {
if comErr, ok := err.(*common_http.Error); ok {
if comErr.Code == http.StatusAccepted {
return digest, nil
}
}
}
return digest, err
}
// PullBlob ...
func (a *adapter) PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error) {
size, blob, err = a.Adapter.PullBlob(repository, digest)
if err != nil && blob != nil {
if size == 0 {
var data []byte
defer blob.Close()
data, err = ioutil.ReadAll(blob)
if err != nil {
return
}
size = int64(len(data))
blob = ioutil.NopCloser(bytes.NewReader(data))
return size, blob, nil
}
}
return
}

8
src/replication/adapter/quayio/adapter_test.go → src/replication/adapter/quay/adapter_test.go
View File

@ -1,4 +1,4 @@
package quayio
package quay
import (
"testing"
@ -9,9 +9,9 @@ import (
)
func getMockAdapter(t *testing.T) adp.Adapter {
factory, _ := adp.GetFactory(model.RegistryTypeQuayio)
factory, _ := adp.GetFactory(model.RegistryTypeQuay)
adapter, err := factory.Create(&model.Registry{
Type: model.RegistryTypeQuayio,
Type: model.RegistryTypeQuay,
URL: "https://quay.io",
})
assert.Nil(t, err)
@ -22,7 +22,7 @@ func TestAdapter_NewAdapter(t *testing.T) {
assert.Nil(t, factory)
assert.NotNil(t, err)
factory, err = adp.GetFactory(model.RegistryTypeQuayio)
factory, err = adp.GetFactory(model.RegistryTypeQuay)
assert.Nil(t, err)
assert.NotNil(t, factory)
}

2
src/replication/adapter/quayio/apikey.go → src/replication/adapter/quay/apikey.go
View File

@ -1,4 +1,4 @@
package quayio
package quay
import (
"fmt"

2
src/replication/adapter/quayio/apikey_test.go → src/replication/adapter/quay/apikey_test.go
View File

@ -1,4 +1,4 @@
package quayio
package quay
import (
"net/http"

21
src/replication/adapter/quay/types.go Normal file
View File

@ -0,0 +1,21 @@
package quay
import (
"fmt"
"strings"
)
type cred struct {
OAuth2Token string `json:"oauth2_token"`
AccountName string `json:"account_name"`
DockerCliPassword string `json:"docker_cli_password"`
}
type orgCreate struct {
Name string `json:"name"`
Email string `json:"email"`
}
func buildOrgURL(endpoint, orgName string) string {
return fmt.Sprintf("%s/api/v1/organization/%s", strings.TrimRight(endpoint, "/"), orgName)
}

12
src/replication/adapter/quayio/types.go
View File

@ -1,12 +0,0 @@
package quayio
import "fmt"
type orgCreate struct {
Name string `json:"name"`
Email string `json:"email"`
}
func buildOrgURL(orgName string) string {
return fmt.Sprintf("https://quay.io/api/v1/organization/%s", orgName)
}

2
src/replication/model/registry.go
View File

@ -29,7 +29,7 @@ const (
RegistryTypeAzureAcr RegistryType = "azure-acr"
RegistryTypeAliAcr RegistryType = "ali-acr"
RegistryTypeJfrogArtifactory RegistryType = "jfrog-artifactory"
RegistryTypeQuayio RegistryType = "quay-io"
RegistryTypeQuay RegistryType = "quay"
RegistryTypeGitLab RegistryType = "gitlab"
RegistryTypeHelmHub RegistryType = "helm-hub"

2
src/replication/replication.go
View File

@ -46,7 +46,7 @@ import (
// register the Jfrog Artifactory adapter
_ "github.com/goharbor/harbor/src/replication/adapter/jfrog"
// register the Quay.io adapter
_ "github.com/goharbor/harbor/src/replication/adapter/quayio"
_ "github.com/goharbor/harbor/src/replication/adapter/quay"
// register the Helm Hub adapter
_ "github.com/goharbor/harbor/src/replication/adapter/helmhub"
// register the GitLab adapter