diff --git a/make/migrations/postgresql/0040_2.1.0_schema.up.sql b/make/migrations/postgresql/0040_2.1.0_schema.up.sql index 698cc3f7c..4fae82bcd 100644 --- a/make/migrations/postgresql/0040_2.1.0_schema.up.sql +++ b/make/migrations/postgresql/0040_2.1.0_schema.up.sql @@ -111,4 +111,7 @@ BEGIN END $$; ALTER TABLE schedule DROP COLUMN IF EXISTS job_id; -ALTER TABLE schedule DROP COLUMN IF EXISTS status; \ No newline at end of file +ALTER TABLE schedule DROP COLUMN IF EXISTS status; + +/*replication quay.io update vendor type*/ +UPDATE registry SET type = 'quay' WHERE type = 'quay-io'; \ No newline at end of file diff --git a/src/jobservice/job/impl/replication/replication.go b/src/jobservice/job/impl/replication/replication.go index aeb818107..c649f4fee 100644 --- a/src/jobservice/job/impl/replication/replication.go +++ b/src/jobservice/job/impl/replication/replication.go @@ -45,7 +45,7 @@ import ( // register the Jfrog Artifactory adapter _ "github.com/goharbor/harbor/src/replication/adapter/jfrog" // register the Quay.io adapter - _ "github.com/goharbor/harbor/src/replication/adapter/quayio" + _ "github.com/goharbor/harbor/src/replication/adapter/quay" // register the Helm Hub adapter _ "github.com/goharbor/harbor/src/replication/adapter/helmhub" // register the GitLab adapter diff --git a/src/replication/adapter/quayio/adapter.go b/src/replication/adapter/quay/adapter.go similarity index 59% rename from src/replication/adapter/quayio/adapter.go rename to src/replication/adapter/quay/adapter.go index 37a97c92a..5cab569ea 100644 --- a/src/replication/adapter/quayio/adapter.go +++ b/src/replication/adapter/quay/adapter.go @@ -1,14 +1,17 @@ -package quayio +package quay import ( "bytes" "encoding/json" "errors" "fmt" + "io" "io/ioutil" "net/http" "strings" + "github.com/goharbor/harbor/src/pkg/registry/auth/basic" + common_http "github.com/goharbor/harbor/src/common/http" "github.com/goharbor/harbor/src/common/http/modifier" "github.com/goharbor/harbor/src/lib/log" @@ -25,31 +28,51 @@ var ( type adapter struct { *native.Adapter - registry *model.Registry - client *common_http.Client + autoCreateNs bool + registry *model.Registry + client *common_http.Client } func init() { - err := adp.RegisterFactory(model.RegistryTypeQuayio, new(factory)) + err := adp.RegisterFactory(model.RegistryTypeQuay, new(factory)) if err != nil { - log.Errorf("failed to register factory for Quay.io: %v", err) + log.Errorf("failed to register factory for Quay: %v", err) return } - log.Infof("the factory of Quay.io adapter was registered") + log.Infof("the factory of Quay adapter was registered") } func newAdapter(registry *model.Registry) (*adapter, error) { - modifiers := []modifier.Modifier{} - var authorizer modifier.Modifier - if registry.Credential != nil && len(registry.Credential.AccessKey) != 0 { - authorizer = NewAPIKeyAuthorizer("Authorization", fmt.Sprintf("Bearer %s", registry.Credential.AccessKey), APIKeyInHeader) + var modifiers []modifier.Modifier + + var ( + autoCreateNs bool + basicAuthorizer, apiKeyAuthorizer modifier.Modifier + ) + + if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 { + var jsonCred cred + err := json.Unmarshal([]byte(registry.Credential.AccessSecret), &jsonCred) + if err != nil { + return nil, err + } + basicAuthorizer = basic.NewAuthorizer(jsonCred.AccountName, jsonCred.DockerCliPassword) + if len(jsonCred.OAuth2Token) != 0 { + autoCreateNs = true + apiKeyAuthorizer = NewAPIKeyAuthorizer("Authorization", fmt.Sprintf("Bearer %s", jsonCred.OAuth2Token), APIKeyInHeader) + } } - if authorizer != nil { - modifiers = append(modifiers, authorizer) + + nativeRegistryAdapter := native.NewAdapterWithAuthorizer(registry, basicAuthorizer) + + if apiKeyAuthorizer != nil { + modifiers = append(modifiers, apiKeyAuthorizer) } + return &adapter{ - Adapter: native.NewAdapterWithAuthorizer(registry, authorizer), - registry: registry, + Adapter: nativeRegistryAdapter, + autoCreateNs: autoCreateNs, + registry: registry, client: common_http.NewClient( &http.Client{ Transport: util.GetHTTPTransport(registry.Insecure), @@ -69,13 +92,22 @@ func (f *factory) Create(r *model.Registry) (adp.Adapter, error) { // AdapterPattern ... func (f *factory) AdapterPattern() *model.AdapterPattern { - return nil + info := &model.AdapterPattern{ + EndpointPattern: model.NewDefaultEndpointPattern(), + CredentialPattern: &model.CredentialPattern{ + AccessKeyType: model.AccessKeyTypeFix, + AccessKeyData: "json_file", + AccessSecretType: model.AccessSecretTypeFile, + AccessSecretData: "", + }, + } + return info } // Info returns information of the registry func (a *adapter) Info() (*model.RegistryInfo, error) { return &model.RegistryInfo{ - Type: model.RegistryTypeQuayio, + Type: model.RegistryTypeQuay, SupportedResourceTypes: []model.ResourceType{ model.ResourceTypeImage, }, @@ -109,6 +141,9 @@ func (a *adapter) HealthCheck() (model.HealthStatus, error) { // PrepareForPush does the prepare work that needed for pushing/uploading the resource // eg: create the namespace or repository func (a *adapter) PrepareForPush(resources []*model.Resource) error { + if !a.autoCreateNs { + return nil + } namespaces := []string{} for _, resource := range resources { if resource == nil { @@ -133,14 +168,14 @@ func (a *adapter) PrepareForPush(resources []*model.Resource) error { Name: namespace, }) if err != nil { - return fmt.Errorf("create namespace '%s' in Quay.io error: %v", namespace, err) + return fmt.Errorf("create namespace '%s' in Quay error: %v", namespace, err) } log.Debugf("namespace %s created", namespace) } return nil } -// createNamespace creates a new namespace in Quay.io +// createNamespace creates a new namespace in Quay func (a *adapter) createNamespace(namespace *model.Namespace) error { ns, err := a.getNamespace(namespace.Name) if err != nil { @@ -149,24 +184,25 @@ func (a *adapter) createNamespace(namespace *model.Namespace) error { // If the namespace already exist, return succeeded directly. if ns != nil { - log.Infof("Namespace %s already exist in Quay.io, skip it.", namespace.Name) + log.Infof("Namespace %s already exist in Quay, skip it.", namespace.Name) return nil } org := &orgCreate{ Name: namespace.Name, - Email: namespace.GetStringMetadata("email", namespace.Name), + Email: fmt.Sprintf("%s@quay.io", namespace.Name), } b, err := json.Marshal(org) if err != nil { return err } - req, err := http.NewRequest(http.MethodPost, buildOrgURL(""), bytes.NewReader(b)) + req, err := http.NewRequest(http.MethodPost, buildOrgURL(a.registry.URL, ""), bytes.NewBuffer(b)) if err != nil { return err } + req.Header.Set("Content-Type", "application/json") resp, err := a.client.Do(req) if err != nil { return err @@ -185,9 +221,9 @@ func (a *adapter) createNamespace(namespace *model.Namespace) error { return fmt.Errorf("%d -- %s", resp.StatusCode, body) } -// getNamespace get namespace from Quay.io, if the namespace not found, two nil would be returned. +// getNamespace get namespace from Quay, if the namespace not found, two nil would be returned. func (a *adapter) getNamespace(namespace string) (*model.Namespace, error) { - req, err := http.NewRequest(http.MethodGet, buildOrgURL(namespace), nil) + req, err := http.NewRequest(http.MethodGet, buildOrgURL(a.registry.URL, namespace), nil) if err != nil { return nil, err } @@ -216,3 +252,35 @@ func (a *adapter) getNamespace(namespace string) (*model.Namespace, error) { Name: namespace, }, nil } + +// PushManifest ... +func (a *adapter) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) { + digest, err := a.Adapter.PushManifest(repository, reference, mediaType, payload) + if err != nil { + if comErr, ok := err.(*common_http.Error); ok { + if comErr.Code == http.StatusAccepted { + return digest, nil + } + } + } + return digest, err +} + +// PullBlob ... +func (a *adapter) PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error) { + size, blob, err = a.Adapter.PullBlob(repository, digest) + if err != nil && blob != nil { + if size == 0 { + var data []byte + defer blob.Close() + data, err = ioutil.ReadAll(blob) + if err != nil { + return + } + size = int64(len(data)) + blob = ioutil.NopCloser(bytes.NewReader(data)) + return size, blob, nil + } + } + return +} diff --git a/src/replication/adapter/quayio/adapter_test.go b/src/replication/adapter/quay/adapter_test.go similarity index 86% rename from src/replication/adapter/quayio/adapter_test.go rename to src/replication/adapter/quay/adapter_test.go index d2d5fb956..3964fec82 100644 --- a/src/replication/adapter/quayio/adapter_test.go +++ b/src/replication/adapter/quay/adapter_test.go @@ -1,4 +1,4 @@ -package quayio +package quay import ( "testing" @@ -9,9 +9,9 @@ import ( ) func getMockAdapter(t *testing.T) adp.Adapter { - factory, _ := adp.GetFactory(model.RegistryTypeQuayio) + factory, _ := adp.GetFactory(model.RegistryTypeQuay) adapter, err := factory.Create(&model.Registry{ - Type: model.RegistryTypeQuayio, + Type: model.RegistryTypeQuay, URL: "https://quay.io", }) assert.Nil(t, err) @@ -22,7 +22,7 @@ func TestAdapter_NewAdapter(t *testing.T) { assert.Nil(t, factory) assert.NotNil(t, err) - factory, err = adp.GetFactory(model.RegistryTypeQuayio) + factory, err = adp.GetFactory(model.RegistryTypeQuay) assert.Nil(t, err) assert.NotNil(t, factory) } diff --git a/src/replication/adapter/quayio/apikey.go b/src/replication/adapter/quay/apikey.go similarity index 98% rename from src/replication/adapter/quayio/apikey.go rename to src/replication/adapter/quay/apikey.go index 8c803a82a..a92e0c201 100644 --- a/src/replication/adapter/quayio/apikey.go +++ b/src/replication/adapter/quay/apikey.go @@ -1,4 +1,4 @@ -package quayio +package quay import ( "fmt" diff --git a/src/replication/adapter/quayio/apikey_test.go b/src/replication/adapter/quay/apikey_test.go similarity index 98% rename from src/replication/adapter/quayio/apikey_test.go rename to src/replication/adapter/quay/apikey_test.go index a5b9c4344..cb2cafe9a 100644 --- a/src/replication/adapter/quayio/apikey_test.go +++ b/src/replication/adapter/quay/apikey_test.go @@ -1,4 +1,4 @@ -package quayio +package quay import ( "net/http" diff --git a/src/replication/adapter/quay/types.go b/src/replication/adapter/quay/types.go new file mode 100644 index 000000000..6d684b0aa --- /dev/null +++ b/src/replication/adapter/quay/types.go @@ -0,0 +1,21 @@ +package quay + +import ( + "fmt" + "strings" +) + +type cred struct { + OAuth2Token string `json:"oauth2_token"` + AccountName string `json:"account_name"` + DockerCliPassword string `json:"docker_cli_password"` +} + +type orgCreate struct { + Name string `json:"name"` + Email string `json:"email"` +} + +func buildOrgURL(endpoint, orgName string) string { + return fmt.Sprintf("%s/api/v1/organization/%s", strings.TrimRight(endpoint, "/"), orgName) +} diff --git a/src/replication/adapter/quayio/types.go b/src/replication/adapter/quayio/types.go deleted file mode 100644 index 393dad058..000000000 --- a/src/replication/adapter/quayio/types.go +++ /dev/null @@ -1,12 +0,0 @@ -package quayio - -import "fmt" - -type orgCreate struct { - Name string `json:"name"` - Email string `json:"email"` -} - -func buildOrgURL(orgName string) string { - return fmt.Sprintf("https://quay.io/api/v1/organization/%s", orgName) -} diff --git a/src/replication/model/registry.go b/src/replication/model/registry.go index 84131251e..96cc0d8e5 100644 --- a/src/replication/model/registry.go +++ b/src/replication/model/registry.go @@ -29,7 +29,7 @@ const ( RegistryTypeAzureAcr RegistryType = "azure-acr" RegistryTypeAliAcr RegistryType = "ali-acr" RegistryTypeJfrogArtifactory RegistryType = "jfrog-artifactory" - RegistryTypeQuayio RegistryType = "quay-io" + RegistryTypeQuay RegistryType = "quay" RegistryTypeGitLab RegistryType = "gitlab" RegistryTypeHelmHub RegistryType = "helm-hub" diff --git a/src/replication/replication.go b/src/replication/replication.go index a678a5272..c40486cb1 100644 --- a/src/replication/replication.go +++ b/src/replication/replication.go @@ -46,7 +46,7 @@ import ( // register the Jfrog Artifactory adapter _ "github.com/goharbor/harbor/src/replication/adapter/jfrog" // register the Quay.io adapter - _ "github.com/goharbor/harbor/src/replication/adapter/quayio" + _ "github.com/goharbor/harbor/src/replication/adapter/quay" // register the Helm Hub adapter _ "github.com/goharbor/harbor/src/replication/adapter/helmhub" // register the GitLab adapter