From f71a110bec7444e64d6e07c8e37d0d085cb2c242 Mon Sep 17 00:00:00 2001 From: cd1989 Date: Wed, 3 Apr 2019 19:20:31 +0800 Subject: [PATCH 1/2] Add registry ping API Signed-off-by: cd1989 --- docs/swagger.yaml | 27 ++++++++++++++++++ src/core/api/harborapi_test.go | 7 +++++ src/core/api/registry.go | 38 +++++++++++++++++++++++++ src/core/api/registry_test.go | 39 ++++++++++++++++++++++++++ src/core/router.go | 1 + src/replication/ng/registry/manager.go | 7 +++-- 6 files changed, 116 insertions(+), 3 deletions(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index eca24ab17..8f05634e7 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2160,6 +2160,33 @@ paths: $ref: '#/responses/UnsupportedMediaType' '500': description: Unexpected internal errors. + /registries/ping: + post: + summary: Ping status of a registry. + description: | + This endpoint checks status of a registry, the registry can be given by ID or URL (together with credential) + parameters: + - name: registry + in: body + description: Registry to ping. + required: true + schema: + $ref: '#/definitions/Registry' + tags: + - Products + responses: + '200': + description: Registry is healthy. + '400': + description: No proper registry information provided. + '401': + description: User need to log in first. + '404': + description: Registry not found (when registry is provided by ID). + '415': + $ref: '#/responses/UnsupportedMediaType' + '500': + description: Unexpected internal errors. '/registries/{id}': put: summary: Update a given registry. diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index 1de53ddfd..0231af95b 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -124,6 +124,7 @@ func init() { beego.Router("/api/repositories/*/signatures", &RepositoryAPI{}, "get:GetSignatures") beego.Router("/api/repositories/top", &RepositoryAPI{}, "get:GetTopRepos") beego.Router("/api/registries", &RegistryAPI{}, "get:List;post:Post") + beego.Router("/api/registries/ping", &RegistryAPI{}, "post:Ping") beego.Router("/api/registries/:id([0-9]+)", &RegistryAPI{}, "get:Get;put:Put;delete:Delete") beego.Router("/api/systeminfo", &SystemInfoAPI{}, "get:GetGeneralInfo") beego.Router("/api/systeminfo/volumes", &SystemInfoAPI{}, "get:GetVolumeInfo") @@ -1175,6 +1176,12 @@ func (a testapi) RegistryCreate(authInfo usrInfo, registry *model.Registry) (int return code, err } +func (a testapi) RegistryPing(authInfo usrInfo, registry *model.Registry) (int, error) { + _sling := sling.New().Base(a.basePath).Post("/api/registries/ping").BodyJSON(registry) + code, _, err := request(_sling, jsonAcceptHeader, authInfo) + return code, err +} + func (a testapi) RegistryDelete(authInfo usrInfo, registryID int64) (int, error) { _sling := sling.New().Base(a.basePath).Delete(fmt.Sprintf("/api/registries/%d", registryID)) code, _, err := request(_sling, jsonAcceptHeader, authInfo) diff --git a/src/core/api/registry.go b/src/core/api/registry.go index 9f8c0ecc5..4e1c81dd9 100644 --- a/src/core/api/registry.go +++ b/src/core/api/registry.go @@ -40,6 +40,44 @@ func (t *RegistryAPI) Prepare() { t.policyCtl = ng.PolicyCtl } +// Ping checks health status of a registry +func (t *RegistryAPI) Ping() { + r := &model.Registry{} + t.DecodeJSONReqAndValidate(r) + + var err error + id := r.ID + if id != 0 { + r, err = t.manager.Get(id) + if err != nil { + log.Errorf("failed to get registry %s: %v", r.Name, err) + t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + return + } + + if r == nil { + t.CustomAbort(http.StatusNotFound, fmt.Sprintf("Registry %d not found", id)) + return + } + } + + if len(r.URL) == 0 || r.Credential == nil { + t.CustomAbort(http.StatusBadRequest, "URL or credential emptry") + return + } + + status, err := registry.CheckHealthStatus(r) + if err != nil { + t.CustomAbort(http.StatusInternalServerError, fmt.Sprintf("Ping registry %s error: %v", r.URL, err)) + return + } + + if status != registry.Healthy { + t.CustomAbort(http.StatusBadRequest, fmt.Sprintf("Ping registry %d failed", r.ID)) + } + return +} + // Get gets a registry by id. func (t *RegistryAPI) Get() { id := t.GetIDFromURL() diff --git a/src/core/api/registry_test.go b/src/core/api/registry_test.go index 7c5d5daad..b6da241c4 100644 --- a/src/core/api/registry_test.go +++ b/src/core/api/registry_test.go @@ -117,6 +117,45 @@ func (suite *RegistrySuite) TestPost() { assert.Equal(http.StatusForbidden, code) } +func (suite *RegistrySuite) TestPing() { + assert := assert.New(suite.T()) + + code, err := suite.testAPI.RegistryPing(*admin, &model.Registry{ + ID: suite.defaultRegistry.ID, + }) + assert.Nil(err) + assert.Equal(http.StatusInternalServerError, code) + + code, err = suite.testAPI.RegistryPing(*admin, &model.Registry{ + ID: -1, + }) + assert.Nil(err) + assert.Equal(http.StatusNotFound, code) + + code, err = suite.testAPI.RegistryPing(*admin, &model.Registry{ + URL: "http://foo.bar", + }) + assert.Nil(err) + assert.Equal(http.StatusBadRequest, code) + + code, err = suite.testAPI.RegistryPing(*admin, &model.Registry{ + URL: "http://foo.bar", + Credential: &model.Credential{ + Type: model.CredentialTypeBasic, + AccessKey: "admin", + AccessSecret: "Harbor12345", + }, + }) + assert.Nil(err) + assert.NotEqual(http.StatusBadRequest, code) + + code, err = suite.testAPI.RegistryPing(*testUser, &model.Registry{ + ID: suite.defaultRegistry.ID, + }) + assert.Nil(err) + assert.Equal(http.StatusForbidden, code) +} + func (suite *RegistrySuite) TestRegistryPut() { assert := assert.New(suite.T()) diff --git a/src/core/router.go b/src/core/router.go index 27f8b15d2..43932370d 100644 --- a/src/core/router.go +++ b/src/core/router.go @@ -132,6 +132,7 @@ func initRouters() { beego.Router("/api/registries", &api.RegistryAPI{}, "get:List;post:Post") beego.Router("/api/registries/:id([0-9]+)", &api.RegistryAPI{}, "get:Get;put:Put;delete:Delete") + beego.Router("/api/registries/ping", &api.RegistryAPI{}, "post:Ping") // we use "0" as the ID of the local Harbor registry, so don't add "([0-9]+)" in the path beego.Router("/api/registries/:id/info", &api.RegistryAPI{}, "get:GetInfo") diff --git a/src/replication/ng/registry/manager.go b/src/replication/ng/registry/manager.go index 85600551f..b147880c3 100644 --- a/src/replication/ng/registry/manager.go +++ b/src/replication/ng/registry/manager.go @@ -183,7 +183,7 @@ func (m *DefaultManager) HealthCheck() error { errCount := 0 for _, r := range registries { - status, err := healthStatus(r) + status, err := CheckHealthStatus(r) if err != nil { log.Warningf("Check health status for %s error: %v", r.URL, err) } @@ -202,7 +202,8 @@ func (m *DefaultManager) HealthCheck() error { return nil } -func healthStatus(r *model.Registry) (HealthStatus, error) { +// CheckHealthStatus checks status of a given registry +func CheckHealthStatus(r *model.Registry) (HealthStatus, error) { // TODO(ChenDe): Support other credential type like OAuth, for the moment, only basic auth is supported. if r.Credential.Type != model.CredentialTypeBasic { return Unknown, fmt.Errorf("unknown credential type '%s', only '%s' supported yet", r.Credential.Type, model.CredentialTypeBasic) @@ -210,7 +211,7 @@ func healthStatus(r *model.Registry) (HealthStatus, error) { // TODO(ChenDe): Support health check for other kinds of registry if r.Type != model.RegistryTypeHarbor { - return Unknown, fmt.Errorf("unknown registry type '%s'", model.RegistryTypeHarbor) + return Unknown, fmt.Errorf("unknown registry type '%s'", r.Type) } transport := util.GetHTTPTransport(r.Insecure) From 5a2d03593f4a2d7ebb2c60fd61bbbd82bae0bc59 Mon Sep 17 00:00:00 2001 From: cd1989 Date: Sat, 6 Apr 2019 18:05:38 +0800 Subject: [PATCH 2/2] Add helth check method to registry adapter Signed-off-by: cd1989 --- src/common/utils/registry/registry.go | 20 ++++++ src/core/api/registry.go | 6 +- src/core/api/registry_test.go | 4 +- src/replication/ng/adapter/adapter.go | 8 +++ src/replication/ng/adapter/image_registry.go | 61 +++++++++++++++++-- src/replication/ng/model/registry.go | 12 ++++ .../ng/operation/controller_test.go | 3 + .../ng/operation/flow/stage_test.go | 3 + src/replication/ng/registry/manager.go | 47 +++----------- 9 files changed, 114 insertions(+), 50 deletions(-) diff --git a/src/common/utils/registry/registry.go b/src/common/utils/registry/registry.go index 692c39382..be26c79fb 100644 --- a/src/common/utils/registry/registry.go +++ b/src/common/utils/registry/registry.go @@ -157,3 +157,23 @@ func (r *Registry) Ping() error { Message: string(b), } } + +// PingSimple checks whether the registry is available. It checks the connectivity and certificate (if TLS enabled) +// only, regardless of credential. +func (r *Registry) PingSimple() error { + err := r.Ping() + if err == nil { + return nil + } + + httpErr, ok := err.(*commonhttp.Error) + if !ok { + return err + } + + if httpErr.Code < 500 { + return nil + } + + return httpErr +} diff --git a/src/core/api/registry.go b/src/core/api/registry.go index 4e1c81dd9..2fa1a174f 100644 --- a/src/core/api/registry.go +++ b/src/core/api/registry.go @@ -61,8 +61,8 @@ func (t *RegistryAPI) Ping() { } } - if len(r.URL) == 0 || r.Credential == nil { - t.CustomAbort(http.StatusBadRequest, "URL or credential emptry") + if len(r.URL) == 0 { + t.CustomAbort(http.StatusBadRequest, "URL can't be emptry") return } @@ -72,7 +72,7 @@ func (t *RegistryAPI) Ping() { return } - if status != registry.Healthy { + if status != model.Healthy { t.CustomAbort(http.StatusBadRequest, fmt.Sprintf("Ping registry %d failed", r.ID)) } return diff --git a/src/core/api/registry_test.go b/src/core/api/registry_test.go index b6da241c4..fbdbe418e 100644 --- a/src/core/api/registry_test.go +++ b/src/core/api/registry_test.go @@ -132,9 +132,7 @@ func (suite *RegistrySuite) TestPing() { assert.Nil(err) assert.Equal(http.StatusNotFound, code) - code, err = suite.testAPI.RegistryPing(*admin, &model.Registry{ - URL: "http://foo.bar", - }) + code, err = suite.testAPI.RegistryPing(*admin, &model.Registry{}) assert.Nil(err) assert.Equal(http.StatusBadRequest, code) diff --git a/src/replication/ng/adapter/adapter.go b/src/replication/ng/adapter/adapter.go index 7bd32365d..3fb70ffbf 100644 --- a/src/replication/ng/adapter/adapter.go +++ b/src/replication/ng/adapter/adapter.go @@ -40,6 +40,8 @@ type Adapter interface { // Get the namespace specified by the name, the returning value should // contain the metadata about the namespace if it has GetNamespace(string) (*model.Namespace, error) + // HealthCheck checks health status of registry + HealthCheck() (model.HealthStatus, error) } // RegisterFactory registers one adapter factory to the registry @@ -67,6 +69,12 @@ func GetFactory(t model.RegistryType) (Factory, error) { return factory, nil } +// HasFactory checks whether there is given type adapter factory +func HasFactory(t model.RegistryType) bool { + _, ok := registry[t] + return ok +} + // ListRegisteredAdapterTypes lists the registered Adapter type func ListRegisteredAdapterTypes() []model.RegistryType { types := []model.RegistryType{} diff --git a/src/replication/ng/adapter/image_registry.go b/src/replication/ng/adapter/image_registry.go index 6ee0f180e..03b606bb1 100644 --- a/src/replication/ng/adapter/image_registry.go +++ b/src/replication/ng/adapter/image_registry.go @@ -28,6 +28,7 @@ import ( common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth" "github.com/goharbor/harbor/src/common/utils/log" registry_pkg "github.com/goharbor/harbor/src/common/utils/registry" + util_registry "github.com/goharbor/harbor/src/common/utils/registry" "github.com/goharbor/harbor/src/common/utils/registry/auth" "github.com/goharbor/harbor/src/replication/ng/model" "github.com/goharbor/harbor/src/replication/ng/util" @@ -54,9 +55,10 @@ type ImageRegistry interface { // DefaultImageRegistry provides a default implementation for interface ImageRegistry type DefaultImageRegistry struct { sync.RWMutex - client *http.Client - url string - clients map[string]*registry_pkg.Repository + registry *model.Registry + client *http.Client + url string + clients map[string]*registry_pkg.Repository } // NewDefaultImageRegistry returns an instance of DefaultImageRegistry @@ -92,9 +94,10 @@ func NewDefaultImageRegistry(registry *model.Registry) *DefaultImageRegistry { Transport: registry_pkg.NewTransport(transport, modifiers...), } return &DefaultImageRegistry{ - client: client, - clients: map[string]*registry_pkg.Repository{}, - url: registry.URL, + client: client, + registry: registry, + clients: map[string]*registry_pkg.Repository{}, + url: registry.URL, } } @@ -134,6 +137,52 @@ func (d *DefaultImageRegistry) create(repository string) (*registry_pkg.Reposito return client, nil } +// HealthCheck checks health status of a registry +func (d *DefaultImageRegistry) HealthCheck() (model.HealthStatus, error) { + if d.registry.Credential == nil || (len(d.registry.Credential.AccessKey) == 0 && len(d.registry.Credential.AccessSecret) == 0) { + return d.pingAnonymously() + } + + // TODO(ChenDe): Support other credential type like OAuth, for the moment, only basic auth is supported. + if d.registry.Credential.Type != model.CredentialTypeBasic { + return model.Unknown, fmt.Errorf("unknown credential type '%s', only '%s' supported yet", d.registry.Credential.Type, model.CredentialTypeBasic) + } + + transport := util.GetHTTPTransport(d.registry.Insecure) + credential := auth.NewBasicAuthCredential(d.registry.Credential.AccessKey, d.registry.Credential.AccessSecret) + authorizer := auth.NewStandardTokenAuthorizer(&http.Client{ + Transport: transport, + }, credential) + registry, err := util_registry.NewRegistry(d.registry.URL, &http.Client{ + Transport: util_registry.NewTransport(transport, authorizer), + }) + if err != nil { + return model.Unknown, err + } + + err = registry.Ping() + if err != nil { + return model.Unhealthy, err + } + return model.Healthy, nil +} + +func (d *DefaultImageRegistry) pingAnonymously() (model.HealthStatus, error) { + registry, err := util_registry.NewRegistry(d.registry.URL, &http.Client{ + Transport: util_registry.NewTransport(util.GetHTTPTransport(d.registry.Insecure)), + }) + if err != nil { + return model.Unknown, err + } + + err = registry.PingSimple() + if err != nil { + return model.Unhealthy, err + } + + return model.Healthy, nil +} + // FetchImages ... func (d *DefaultImageRegistry) FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) { return nil, errors.New("not implemented") diff --git a/src/replication/ng/model/registry.go b/src/replication/ng/model/registry.go index 853d71c96..eca9e03b5 100644 --- a/src/replication/ng/model/registry.go +++ b/src/replication/ng/model/registry.go @@ -56,6 +56,18 @@ type Credential struct { AccessSecret string `json:"access_secret"` } +// HealthStatus describes whether a target is healthy or not +type HealthStatus string + +const ( + // Healthy indicates registry is healthy + Healthy = "healthy" + // Unhealthy indicates registry is unhealthy + Unhealthy = "unhealthy" + // Unknown indicates health status of registry is unknown + Unknown = "unknown" +) + // TODO add validation for Registry // Registry keeps the related info of registry diff --git a/src/replication/ng/operation/controller_test.go b/src/replication/ng/operation/controller_test.go index 35efb240f..dbe599352 100644 --- a/src/replication/ng/operation/controller_test.go +++ b/src/replication/ng/operation/controller_test.go @@ -135,6 +135,9 @@ func (f *fakedAdapter) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace func (f *fakedAdapter) CreateNamespace(*model.Namespace) error { return nil } +func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) { + return model.Healthy, nil +} func (f *fakedAdapter) GetNamespace(ns string) (*model.Namespace, error) { var namespace *model.Namespace if ns == "library" { diff --git a/src/replication/ng/operation/flow/stage_test.go b/src/replication/ng/operation/flow/stage_test.go index ecc134d8d..0d31e6d40 100644 --- a/src/replication/ng/operation/flow/stage_test.go +++ b/src/replication/ng/operation/flow/stage_test.go @@ -52,6 +52,9 @@ func (f *fakedAdapter) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace func (f *fakedAdapter) CreateNamespace(*model.Namespace) error { return nil } +func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) { + return model.Healthy, nil +} func (f *fakedAdapter) GetNamespace(ns string) (*model.Namespace, error) { var namespace *model.Namespace if ns == "library" { diff --git a/src/replication/ng/registry/manager.go b/src/replication/ng/registry/manager.go index b147880c3..6382d674e 100644 --- a/src/replication/ng/registry/manager.go +++ b/src/replication/ng/registry/manager.go @@ -16,32 +16,16 @@ package registry import ( "fmt" - "net/http" - - "github.com/goharbor/harbor/src/replication/ng/util" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/common/utils/registry" - "github.com/goharbor/harbor/src/common/utils/registry/auth" + "github.com/goharbor/harbor/src/replication/ng/adapter" "github.com/goharbor/harbor/src/replication/ng/config" "github.com/goharbor/harbor/src/replication/ng/dao" "github.com/goharbor/harbor/src/replication/ng/dao/models" "github.com/goharbor/harbor/src/replication/ng/model" ) -// HealthStatus describes whether a target is healthy or not -type HealthStatus string - -const ( - // Healthy indicates target is healthy - Healthy = "healthy" - // Unhealthy indicates target is unhealthy - Unhealthy = "unhealthy" - // Unknown indicates health status of target is unknown - Unknown = "unknown" -) - // Manager defines the methods that a target manager should implement type Manager interface { // Add new registry @@ -203,35 +187,22 @@ func (m *DefaultManager) HealthCheck() error { } // CheckHealthStatus checks status of a given registry -func CheckHealthStatus(r *model.Registry) (HealthStatus, error) { - // TODO(ChenDe): Support other credential type like OAuth, for the moment, only basic auth is supported. - if r.Credential.Type != model.CredentialTypeBasic { - return Unknown, fmt.Errorf("unknown credential type '%s', only '%s' supported yet", r.Credential.Type, model.CredentialTypeBasic) +func CheckHealthStatus(r *model.Registry) (model.HealthStatus, error) { + if !adapter.HasFactory(r.Type) { + return model.Unknown, fmt.Errorf("no adapter factory for type '%s' registered", r.Type) } - // TODO(ChenDe): Support health check for other kinds of registry - if r.Type != model.RegistryTypeHarbor { - return Unknown, fmt.Errorf("unknown registry type '%s'", r.Type) - } - - transport := util.GetHTTPTransport(r.Insecure) - credential := auth.NewBasicAuthCredential(r.Credential.AccessKey, r.Credential.AccessSecret) - authorizer := auth.NewStandardTokenAuthorizer(&http.Client{ - Transport: transport, - }, credential) - registry, err := registry.NewRegistry(r.URL, &http.Client{ - Transport: registry.NewTransport(transport, authorizer), - }) + factory, err := adapter.GetFactory(r.Type) if err != nil { - return Unknown, err + return model.Unknown, fmt.Errorf("get adaper for type '%s' error: %v", r.Type, err) } - err = registry.Ping() + rAdapter, err := factory(r) if err != nil { - return Unhealthy, err + return model.Unknown, fmt.Errorf("generate '%s' type adapter form factory error: %v", r.Type, err) } - return Healthy, nil + return rAdapter.HealthCheck() } // decrypt checks whether access secret is set in the registry, if so, decrypt it.