From 107e468b6017c74ca899ef90414789d7a0d90b01 Mon Sep 17 00:00:00 2001 From: stonezdj Date: Thu, 11 Mar 2021 20:25:51 +0800 Subject: [PATCH] Refactor configure api to new programming model Changes include: 1. Move core/config to controller/config 2. Change the job_service and gcreadonly to depends on lib/config instead of core/config 3. Move the config related dao, manager and driver to pkg/config 4. Adjust the invocation of the config API, most of then should provide a context parameter, when accessing system config, you can call it with background context, when accessing user config, the context should provide orm.Context Signed-off-by: stonezdj --- api/v2.0/legacy_swagger.yaml | 40 -- api/v2.0/swagger.yaml | 272 ++++++++++ src/chartserver/handler_utility.go | 2 +- src/cmd/exporter/main.go | 2 +- src/cmd/standalone-db-migrator/main.go | 2 +- src/common/config/encrypt/keyprovider.go | 48 -- src/common/config/encrypt/keyprovider_test.go | 44 -- src/common/config/keyprovider_test.go | 54 -- src/common/config/manager.go | 256 --------- src/common/config/manager_test.go | 141 ----- src/common/config/store/driver/db_test.go | 85 --- src/common/config/store/driver/driver.go | 10 - src/common/config/store/driver/rest.go | 45 -- src/common/config/store/store_test.go | 93 ---- src/common/const.go | 5 +- src/common/dao/base.go | 8 - src/common/dao/config.go | 77 --- src/common/dao/config_test.go | 8 +- src/common/dao/dao_test.go | 104 +--- src/common/dao/label.go | 3 +- src/common/dao/pgsql.go | 2 +- src/common/dao/pro_meta.go | 16 +- src/common/dao/project.go | 6 +- src/common/dao/project/projectmember.go | 8 +- src/common/dao/project/projectmember_test.go | 4 +- src/common/dao/repository.go | 6 +- src/common/dao/testutils.go | 2 +- src/common/dao/user.go | 22 +- src/common/job/client.go | 2 +- src/common/models/base.go | 1 - src/common/models/database.go | 47 ++ src/common/utils/utils.go | 10 + src/controller/config/controller.go | 217 ++++++++ src/controller/config/systemcfg.go | 250 +++++++++ src/{core => controller}/config/test/ca.crt | 0 src/controller/config/test/controller_test.go | 145 ++++++ src/controller/config/userconfig.go | 282 ++++++++++ .../config/userconfig_test.go} | 61 +-- src/controller/event/handler/util/util.go | 9 +- .../event/handler/util/util_test.go | 10 +- .../handler/webhook/artifact/replication.go | 5 +- .../webhook/artifact/replication_test.go | 3 +- .../handler/webhook/artifact/retention.go | 7 +- .../webhook/artifact/retention_test.go | 11 +- .../event/handler/webhook/chart/chart.go | 2 +- .../event/handler/webhook/chart/chart_test.go | 10 +- .../event/handler/webhook/quota/quota_test.go | 2 +- .../event/handler/webhook/scan/scan_test.go | 2 +- src/controller/gc/callback.go | 4 +- src/controller/ldap/controller.go | 24 +- src/controller/ldap/controller_test.go | 2 +- src/controller/p2p/preheat/controllor_test.go | 2 +- src/controller/p2p/preheat/enforcer.go | 5 +- src/controller/proxy/local.go | 2 +- .../quota/driver/project/project.go | 15 +- src/controller/robot/controller.go | 8 +- src/controller/robot/controller_test.go | 8 +- src/controller/robot/model_test.go | 3 +- src/controller/scan/base_controller.go | 2 +- src/controller/scan/base_controller_test.go | 2 +- src/controller/systeminfo/controller.go | 10 +- src/controller/systeminfo/controller_test.go | 5 +- src/controller/tag/controller_test.go | 4 +- src/core/api/base.go | 2 +- src/core/api/chart_repository.go | 2 +- src/core/api/config.go | 191 ------- src/core/api/config_test.go | 127 ----- src/core/api/email.go | 8 +- src/core/api/harborapi_test.go | 85 +-- src/core/api/health.go | 2 +- src/core/api/internal.go | 37 +- src/core/api/projectmember.go | 5 +- src/core/api/user.go | 7 +- src/core/api/user_test.go | 2 +- src/core/api/usergroup.go | 5 +- src/core/auth/authenticator.go | 7 +- src/core/auth/authproxy/auth.go | 10 +- src/core/auth/authproxy/auth_test.go | 15 +- src/core/auth/db/db_test.go | 5 +- src/core/auth/ldap/ldap.go | 17 +- src/core/auth/ldap/ldap_test.go | 4 +- src/core/auth/uaa/uaa.go | 5 +- src/core/auth/uaa/uaa_test.go | 2 +- src/core/config/config.go | 491 ------------------ src/core/controllers/authproxy_redirect.go | 5 +- src/core/controllers/base.go | 9 +- src/core/controllers/controllers_test.go | 8 +- src/core/controllers/oidc.go | 7 +- src/core/main.go | 4 +- src/core/service/token/authutils.go | 6 +- src/core/service/token/creator.go | 4 +- src/core/service/token/token_test.go | 7 +- src/core/utils/job.go | 3 +- src/jobservice/job/impl/context.go | 16 +- src/jobservice/job/impl/context_test.go | 13 +- .../job/impl/gcreadonly/garbage_collection.go | 31 +- .../gcreadonly/garbage_collection_test.go | 20 +- src/jobservice/main.go | 11 +- src/lib/config/config.go | 48 ++ src/{common => lib}/config/context.go | 21 +- .../config/metadata/metadata.go | 0 .../config/metadata/metadata_test.go | 0 .../config/metadata/metadatalist.go | 116 +++-- src/{common => lib}/config/metadata/type.go | 0 .../config/metadata/type_test.go | 0 src/{common => lib}/config/metadata/value.go | 0 .../config/metadata/value_test.go | 0 src/lib/config/metadata/yaml/genyaml.go | 127 +++++ src/lib/config/model.go | 24 + .../config.go => lib/config/models/model.go} | 67 +-- src/lib/orm/error.go | 5 +- src/lib/orm/orm.go | 10 +- src/lib/orm/{ => test}/orm_test.go | 23 +- src/pkg/authproxy/http.go | 5 +- src/pkg/authproxy/http_test.go | 2 +- .../store/driver => pkg/config/db}/cache.go | 36 +- .../driver => pkg/config/db}/cache_test.go | 38 +- src/pkg/config/db/dao/dao.go | 106 ++++ src/pkg/config/db/dao/dao_test.go | 132 +++++ .../store/driver => pkg/config/db}/db.go | 40 +- src/pkg/config/db/db_test.go | 79 +++ src/pkg/config/db/manager.go | 46 ++ src/pkg/config/db/manager_test.go | 194 +++++++ src/pkg/config/inmemory/manager.go | 67 +++ src/pkg/config/inmemory/manager_test.go | 34 ++ src/pkg/config/manager.go | 186 +++++++ src/pkg/config/rest/manager.go | 53 ++ src/pkg/config/rest/rest.go | 70 +++ .../driver => pkg/config/rest}/rest_test.go | 27 +- src/pkg/config/store/driver.go | 26 + src/{common => pkg}/config/store/store.go | 25 +- src/{common/config => pkg}/encrypt/encrypt.go | 20 +- .../config => pkg}/encrypt/encrypt_test.go | 14 + .../config => pkg/encrypt}/keyprovider.go | 22 +- src/pkg/encrypt/keyprovider_test.go | 44 ++ src/pkg/ldap/ldap.go | 2 + src/pkg/notification/hook/hook.go | 2 +- src/pkg/oidc/helper.go | 26 +- src/pkg/oidc/helper_test.go | 30 +- src/pkg/oidc/secret.go | 2 +- src/pkg/quota/dao/util.go | 2 +- src/pkg/reg/manager.go | 2 +- src/pkg/registry/client.go | 2 +- src/pkg/scan/job.go | 4 +- src/pkg/signature/manager.go | 4 +- src/pkg/signature/manager_test.go | 4 +- src/pkg/signature/notary/helper.go | 11 +- src/pkg/signature/notary/helper_test.go | 9 +- src/pkg/task/task.go | 2 +- src/pkg/token/options.go | 2 +- src/pkg/token/token_test.go | 2 +- src/server/middleware/blob/head_blob.go | 2 +- src/server/middleware/blob/util.go | 2 +- src/server/middleware/csrf/csrf.go | 2 +- src/server/middleware/csrf/csrf_test.go | 4 +- src/server/middleware/metric/metric.go | 2 +- src/server/middleware/readonly/readonly.go | 5 +- src/server/middleware/security/auth_proxy.go | 5 +- .../middleware/security/auth_proxy_test.go | 7 +- src/server/middleware/security/idtoken.go | 4 +- src/server/middleware/security/robot2.go | 6 +- src/server/middleware/security/robot2_test.go | 4 +- src/server/middleware/security/secret.go | 2 +- src/server/middleware/security/secret_test.go | 5 +- src/server/middleware/security/security.go | 5 +- .../middleware/security/v2_token_test.go | 8 +- src/server/middleware/session/session.go | 2 +- src/server/middleware/session/session_test.go | 2 +- src/server/middleware/v2auth/auth.go | 2 +- src/server/middleware/v2auth/auth_test.go | 4 +- src/server/registry/proxy.go | 2 +- src/server/route.go | 3 +- src/server/v2.0/handler/base.go | 9 + src/server/v2.0/handler/config.go | 112 ++++ src/server/v2.0/handler/gc.go | 2 +- src/server/v2.0/handler/handler.go | 1 + src/server/v2.0/handler/project.go | 10 +- src/server/v2.0/handler/robot.go | 6 +- src/server/v2.0/handler/search.go | 2 +- src/server/v2.0/route/legacy.go | 4 +- src/testing/controller/controller.go | 1 + src/testing/suite.go | 2 +- tests/apitests/python/library/base.py | 390 +++++++------- .../apitests/python/library/configurations.py | 43 +- tests/apitests/python/test_user_group.py | 12 +- tests/apitests/python/testutils.py | 16 +- 186 files changed, 3528 insertions(+), 2683 deletions(-) delete mode 100644 src/common/config/encrypt/keyprovider.go delete mode 100644 src/common/config/encrypt/keyprovider_test.go delete mode 100644 src/common/config/keyprovider_test.go delete mode 100644 src/common/config/manager.go delete mode 100644 src/common/config/manager_test.go delete mode 100644 src/common/config/store/driver/db_test.go delete mode 100644 src/common/config/store/driver/driver.go delete mode 100644 src/common/config/store/driver/rest.go delete mode 100644 src/common/config/store/store_test.go delete mode 100644 src/common/dao/config.go create mode 100644 src/common/models/database.go create mode 100644 src/controller/config/controller.go create mode 100644 src/controller/config/systemcfg.go rename src/{core => controller}/config/test/ca.crt (100%) create mode 100644 src/controller/config/test/controller_test.go create mode 100755 src/controller/config/userconfig.go rename src/{core/config/config_test.go => controller/config/userconfig_test.go} (85%) delete mode 100644 src/core/api/config.go delete mode 100644 src/core/api/config_test.go delete mode 100755 src/core/config/config.go create mode 100644 src/lib/config/config.go rename src/{common => lib}/config/context.go (50%) rename src/{common => lib}/config/metadata/metadata.go (100%) rename src/{common => lib}/config/metadata/metadata_test.go (100%) rename src/{common => lib}/config/metadata/metadatalist.go (70%) rename src/{common => lib}/config/metadata/type.go (100%) rename src/{common => lib}/config/metadata/type_test.go (100%) rename src/{common => lib}/config/metadata/value.go (100%) rename src/{common => lib}/config/metadata/value_test.go (100%) create mode 100644 src/lib/config/metadata/yaml/genyaml.go create mode 100644 src/lib/config/model.go rename src/{common/models/config.go => lib/config/models/model.go} (55%) rename src/lib/orm/{ => test}/orm_test.go (92%) rename src/{common/config/store/driver => pkg/config/db}/cache.go (50%) rename src/{common/config/store/driver => pkg/config/db}/cache_test.go (51%) create mode 100644 src/pkg/config/db/dao/dao.go create mode 100644 src/pkg/config/db/dao/dao_test.go rename src/{common/config/store/driver => pkg/config/db}/db.go (62%) create mode 100644 src/pkg/config/db/db_test.go create mode 100644 src/pkg/config/db/manager.go create mode 100644 src/pkg/config/db/manager_test.go create mode 100644 src/pkg/config/inmemory/manager.go create mode 100644 src/pkg/config/inmemory/manager_test.go create mode 100644 src/pkg/config/manager.go create mode 100644 src/pkg/config/rest/manager.go create mode 100644 src/pkg/config/rest/rest.go rename src/{common/config/store/driver => pkg/config/rest}/rest_test.go (65%) create mode 100644 src/pkg/config/store/driver.go rename src/{common => pkg}/config/store/store.go (86%) rename src/{common/config => pkg}/encrypt/encrypt.go (71%) rename src/{common/config => pkg}/encrypt/encrypt_test.go (58%) rename src/{common/config => pkg/encrypt}/keyprovider.go (62%) create mode 100644 src/pkg/encrypt/keyprovider_test.go create mode 100644 src/server/v2.0/handler/config.go diff --git a/api/v2.0/legacy_swagger.yaml b/api/v2.0/legacy_swagger.yaml index 9dae1749f..40d16888c 100644 --- a/api/v2.0/legacy_swagger.yaml +++ b/api/v2.0/legacy_swagger.yaml @@ -1008,46 +1008,6 @@ paths: description: Only admin has this authority. '500': description: Unexpected internal errors. - /configurations: - get: - summary: Get system configurations. - description: | - This endpoint is for retrieving system configurations that only provides for admin user. - tags: - - Products - responses: - '200': - description: Get system configurations successfully. The response body is a map. - schema: - $ref: '#/definitions/ConfigurationsResponse' - '401': - description: User need to log in first.ß - '403': - description: User does not have permission of admin role. - '500': - description: Unexpected internal errors. - put: - summary: Modify system configurations. - description: | - This endpoint is for modifying system configurations that only provides for admin user. - tags: - - Products - parameters: - - name: configurations - in: body - required: true - schema: - $ref: '#/definitions/Configurations' - description: 'The configuration map can contain a subset of the attributes of the schema, which are to be updated.' - responses: - '200': - description: Modify system configurations successfully. - '401': - description: User need to log in first. - '403': - description: User does not have permission of admin role. - '500': - description: Unexpected internal errors. /email/ping: post: summary: Test connection and authentication with email server. diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 8450299e4..9857af5de 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -163,6 +163,69 @@ paths: $ref: '#/responses/403' '500': $ref: '#/responses/500' + /internalconfig: + get: + summary: Get internal configurations. + description: | + This endpoint is for retrieving system configurations that only provides for internal api call. + tags: + - configure + parameters: + - $ref: '#/parameters/requestId' + responses: + '200': + description: Get system configurations successfully. The response body is a map. + schema: + $ref: '#/definitions/InternalConfigurationsResponse' + '401': + description: User need to log in first. + '403': + description: User does not have permission of admin role. + '500': + description: Unexpected internal errors. + /configurations: + get: + summary: Get system configurations. + description: | + This endpoint is for retrieving system configurations that only provides for admin user. + tags: + - configure + parameters: + - $ref: '#/parameters/requestId' + responses: + '200': + description: Get system configurations successfully. The response body is a map. + schema: + $ref: '#/definitions/ConfigurationsResponse' + '401': + description: User need to log in first.ß + '403': + description: User does not have permission of admin role. + '500': + description: Unexpected internal errors. + put: + summary: Modify system configurations. + description: | + This endpoint is for modifying system configurations that only provides for admin user. + tags: + - configure + parameters: + - $ref: '#/parameters/requestId' + - name: configurations + in: body + required: true + schema: + $ref: '#/definitions/Configurations' + description: 'The configuration map can contain a subset of the attributes of the schema, which are to be updated.' + responses: + '200': + description: Modify system configurations successfully. + '401': + description: User need to log in first. + '403': + description: User does not have permission of admin role. + '500': + description: Unexpected internal errors. /projects: get: summary: List projects @@ -6635,3 +6698,212 @@ definitions: type: string description: The webhook job update time. format: date-time + InternalConfigurationsResponse: + type: object + x-go-type: + type: InternalCfg + import: + package: "github.com/goharbor/harbor/src/lib/config" + ConfigurationsResponse: + type: object + properties: + auth_mode: + $ref: '#/definitions/StringConfigItem' + description: The auth mode of current system, such as "db_auth", "ldap_auth", "oidc_auth" + email_from: + $ref: '#/definitions/StringConfigItem' + description: The sender name for Email notification. + email_host: + $ref: '#/definitions/StringConfigItem' + description: The hostname of SMTP server that sends Email notification. + email_identity: + $ref: '#/definitions/StringConfigItem' + description: By default it's empty so the email_username is picked + email_insecure: + $ref: '#/definitions/BoolConfigItem' + description: Whether or not the certificate will be verified when Harbor tries to access the email server. + email_port: + $ref: '#/definitions/IntegerConfigItem' + description: The port of SMTP server + email_ssl: + $ref: '#/definitions/BoolConfigItem' + description: When it''s set to true the system will access Email server via TLS by default. If it''s set to false, it still will handle "STARTTLS" from server side. + email_username: + $ref: '#/definitions/StringConfigItem' + description: The username for authenticate against SMTP server + ldap_base_dn: + $ref: '#/definitions/StringConfigItem' + description: The Base DN for LDAP binding. + ldap_filter: + $ref: '#/definitions/StringConfigItem' + description: The filter for LDAP search + ldap_group_base_dn: + $ref: '#/definitions/StringConfigItem' + description: The base DN to search LDAP group. + ldap_group_admin_dn: + $ref: '#/definitions/StringConfigItem' + description: Specify the ldap group which have the same privilege with Harbor admin + ldap_group_attribute_name: + $ref: '#/definitions/StringConfigItem' + description: The attribute which is used as identity of the LDAP group, default is cn.' + ldap_group_search_filter: + $ref: '#/definitions/StringConfigItem' + description: The filter to search the ldap group + ldap_group_search_scope: + $ref: '#/definitions/IntegerConfigItem' + description: The scope to search ldap group. ''0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE'' + ldap_scope: + $ref: '#/definitions/IntegerConfigItem' + description: The scope to search ldap users,'0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE' + ldap_search_dn: + $ref: '#/definitions/StringConfigItem' + description: The DN of the user to do the search. + ldap_timeout: + $ref: '#/definitions/IntegerConfigItem' + description: Timeout in seconds for connection to LDAP server + ldap_uid: + $ref: '#/definitions/StringConfigItem' + description: The attribute which is used as identity for the LDAP binding, such as "CN" or "SAMAccountname" + ldap_url: + $ref: '#/definitions/StringConfigItem' + description: The URL of LDAP server + ldap_verify_cert: + $ref: '#/definitions/BoolConfigItem' + description: Whether verify your OIDC server certificate, disable it if your OIDC server is hosted via self-hosted certificate. + ldap_group_membership_attribute: + $ref: '#/definitions/StringConfigItem' + description: The user attribute to identify the group membership + project_creation_restriction: + $ref: '#/definitions/StringConfigItem' + description: Indicate who can create projects, it could be ''adminonly'' or ''everyone''. + read_only: + $ref: '#/definitions/BoolConfigItem' + description: The flag to indicate whether Harbor is in readonly mode. + self_registration: + $ref: '#/definitions/BoolConfigItem' + description: Whether the Harbor instance supports self-registration. If it''s set to false, admin need to add user to the instance. + token_expiration: + $ref: '#/definitions/IntegerConfigItem' + description: The expiration time of the token for internal Registry, in minutes. + uaa_client_id: + $ref: '#/definitions/StringConfigItem' + description: The client id of UAA + uaa_client_secret: + $ref: '#/definitions/StringConfigItem' + description: The client secret of the UAA + uaa_endpoint: + $ref: '#/definitions/StringConfigItem' + description: The endpoint of the UAA + uaa_verify_cert: + $ref: '#/definitions/BoolConfigItem' + description: Verify the certificate in UAA server + http_authproxy_endpoint: + $ref: '#/definitions/StringConfigItem' + description: The endpoint of the HTTP auth + http_authproxy_tokenreview_endpoint: + $ref: '#/definitions/StringConfigItem' + description: The token review endpoint + http_authproxy_admin_groups: + $ref: '#/definitions/StringConfigItem' + description: The group which has the harbor admin privileges + http_authproxy_admin_usernames: + $ref: '#/definitions/StringConfigItem' + description: The usernames which has the harbor admin privileges + http_authproxy_verify_cert: + $ref: '#/definitions/BoolConfigItem' + description: Verify the HTTP auth provider's certificate + http_authproxy_skip_search: + $ref: '#/definitions/BoolConfigItem' + description: Search user before onboard + http_authproxy_server_certificate: + $ref: '#/definitions/StringConfigItem' + description: The certificate of the HTTP auth provider + oidc_name: + $ref: '#/definitions/StringConfigItem' + description: The OIDC provider name + oidc_endpoint: + $ref: '#/definitions/StringConfigItem' + description: The endpoint of the OIDC provider + oidc_client_id: + $ref: '#/definitions/StringConfigItem' + description: The client ID of the OIDC provider + oidc_groups_claim: + $ref: '#/definitions/StringConfigItem' + description: The attribute claims the group name + oidc_admin_group: + $ref: '#/definitions/StringConfigItem' + description: The OIDC group which has the harbor admin privileges + oidc_scope: + $ref: '#/definitions/StringConfigItem' + description: The scope of the OIDC provider + oidc_user_claim: + $ref: '#/definitions/StringConfigItem' + description: The attribute claims the username + oidc_verify_cert: + $ref: '#/definitions/BoolConfigItem' + description: Verify the OIDC provider's certificate' + oidc_auto_onboard: + $ref: '#/definitions/BoolConfigItem' + description: Auto onboard the OIDC user + oidc_extra_redirect_parms: + $ref: '#/definitions/StringConfigItem' + description: Extra parameters to add when redirect request to OIDC provider + robot_token_duration: + $ref: '#/definitions/IntegerConfigItem' + description: The robot account token duration in days + robot_name_prefix: + $ref: '#/definitions/StringConfigItem' + description: The rebot account name prefix + notification_enable: + $ref: '#/definitions/BoolConfigItem' + description: Enable notification + quota_per_project_enable: + $ref: '#/definitions/BoolConfigItem' + description: Enable quota per project + storage_per_project: + $ref: '#/definitions/IntegerConfigItem' + description: The storage quota per project + scan_all_policy: + type: object + properties: + type: + type: string + description: 'The type of scan all policy, currently the valid values are "none" and "daily"' + parameter: + type: object + properties: + daily_time: + type: integer + description: 'The offset in seconds of UTC 0 o''clock, only valid when the policy type is "daily"' + description: 'The parameters of the policy, the values are dependant on the type of the policy.' + Configurations: + type: object + additionalProperties: + type: object + StringConfigItem: + type: object + properties: + value: + type: string + description: The string value of current config item + editable: + type: boolean + description: The configure item can be updated or not + BoolConfigItem: + type: object + properties: + value: + type: boolean + description: The boolean value of current config item + editable: + type: boolean + description: The configure item can be updated or not + IntegerConfigItem: + type: object + properties: + value: + type: integer + description: The integer value of current config item + editable: + type: boolean + description: The configure item can be updated or not diff --git a/src/chartserver/handler_utility.go b/src/chartserver/handler_utility.go index bea908836..767b17d01 100644 --- a/src/chartserver/handler_utility.go +++ b/src/chartserver/handler_utility.go @@ -2,6 +2,7 @@ package chartserver import ( "fmt" + "github.com/goharbor/harbor/src/controller/config" "path" "strings" "sync" @@ -9,7 +10,6 @@ import ( "github.com/goharbor/harbor/src/lib/errors" "helm.sh/helm/v3/cmd/helm/search" - "github.com/goharbor/harbor/src/core/config" hlog "github.com/goharbor/harbor/src/lib/log" ) diff --git a/src/cmd/exporter/main.go b/src/cmd/exporter/main.go index f3376276f..8b284d294 100644 --- a/src/cmd/exporter/main.go +++ b/src/cmd/exporter/main.go @@ -1,6 +1,7 @@ package main import ( + "github.com/goharbor/harbor/src/common/models" "net/http" "os" "strings" @@ -11,7 +12,6 @@ import ( "github.com/goharbor/harbor/src/common/dao" commonthttp "github.com/goharbor/harbor/src/common/http" - "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/exporter" ) diff --git a/src/cmd/standalone-db-migrator/main.go b/src/cmd/standalone-db-migrator/main.go index db429423d..96482327f 100644 --- a/src/cmd/standalone-db-migrator/main.go +++ b/src/cmd/standalone-db-migrator/main.go @@ -1,11 +1,11 @@ package main import ( + "github.com/goharbor/harbor/src/common/models" "os" "strconv" "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/migration" ) diff --git a/src/common/config/encrypt/keyprovider.go b/src/common/config/encrypt/keyprovider.go deleted file mode 100644 index 0f6c6ced7..000000000 --- a/src/common/config/encrypt/keyprovider.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package encrypt - -import ( - "io/ioutil" -) - -// KeyProvider provides the key used to encrypt and decrypt attrs -type KeyProvider interface { - // Get returns the key - // params can be used to pass parameters in different implements - Get(params map[string]interface{}) (string, error) -} - -// FileKeyProvider reads key from file -type FileKeyProvider struct { - path string -} - -// NewFileKeyProvider returns an instance of FileKeyProvider -// path: where the key should be read from -func NewFileKeyProvider(path string) KeyProvider { - return &FileKeyProvider{ - path: path, - } -} - -// Get returns the key read from file -func (f *FileKeyProvider) Get(params map[string]interface{}) (string, error) { - b, err := ioutil.ReadFile(f.path) - if err != nil { - return "", err - } - return string(b), nil -} diff --git a/src/common/config/encrypt/keyprovider_test.go b/src/common/config/encrypt/keyprovider_test.go deleted file mode 100644 index 8ac93962b..000000000 --- a/src/common/config/encrypt/keyprovider_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package encrypt - -import ( - "io/ioutil" - "os" - "testing" -) - -func TestGetOfFileKeyProvider(t *testing.T) { - path := "/tmp/key" - key := "key_content" - - if err := ioutil.WriteFile(path, []byte(key), 0777); err != nil { - t.Errorf("failed to write to file %s: %v", path, err) - return - } - defer os.Remove(path) - - provider := NewFileKeyProvider(path) - k, err := provider.Get(nil) - if err != nil { - t.Errorf("failed to get key from the file provider: %v", err) - return - } - - if k != key { - t.Errorf("unexpected key: %s != %s", k, key) - return - } -} diff --git a/src/common/config/keyprovider_test.go b/src/common/config/keyprovider_test.go deleted file mode 100644 index c3a025ff2..000000000 --- a/src/common/config/keyprovider_test.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "github.com/stretchr/testify/assert" - "io/ioutil" - "os" - "testing" -) - -func TestGetOfFileKeyProvider(t *testing.T) { - path := "/tmp/key" - key := "key_content" - - if err := ioutil.WriteFile(path, []byte(key), 0777); err != nil { - t.Errorf("failed to write to file %s: %v", path, err) - return - } - defer os.Remove(path) - - provider := NewFileKeyProvider(path) - k, err := provider.Get(nil) - if err != nil { - t.Errorf("failed to get key from the file provider: %v", err) - return - } - - if k != key { - t.Errorf("unexpected key: %s != %s", k, key) - return - } -} - -func TestPresetKeyProvider(t *testing.T) { - kp := &PresetKeyProvider{ - Key: "mykey", - } - k, err := kp.Get(nil) - assert.Nil(t, err) - assert.Equal(t, "mykey", k) -} diff --git a/src/common/config/manager.go b/src/common/config/manager.go deleted file mode 100644 index 3682b2005..000000000 --- a/src/common/config/manager.go +++ /dev/null @@ -1,256 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "fmt" - "os" - "sync" - - "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/config/metadata" - "github.com/goharbor/harbor/src/common/config/store" - "github.com/goharbor/harbor/src/common/config/store/driver" - "github.com/goharbor/harbor/src/common/http/modifier/auth" - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/utils" - "github.com/goharbor/harbor/src/lib/cache" - "github.com/goharbor/harbor/src/lib/log" -) - -// CfgManager ... Configure Manager -type CfgManager struct { - store *store.ConfigStore -} - -// NewDBCfgManager - create DB config manager -func NewDBCfgManager() *CfgManager { - cfgDriver := (driver.Driver)(&driver.Database{}) - - if cache.Default() != nil { - log.Debug("create DB config manager with cache enabled") - cfgDriver = driver.NewCacheDriver(cache.Default(), cfgDriver) - } - - manager := &CfgManager{store: store.NewConfigStore(cfgDriver)} - // load default value - manager.loadDefault() - // load system config from env - manager.loadSystemConfigFromEnv() - return manager -} - -// NewRESTCfgManager - create REST config manager -func NewRESTCfgManager(configURL, secret string) *CfgManager { - secAuth := auth.NewSecretAuthorizer(secret) - manager := &CfgManager{store: store.NewConfigStore(driver.NewRESTDriver(configURL, secAuth))} - return manager -} - -// InMemoryDriver driver for unit testing -type InMemoryDriver struct { - sync.Mutex - cfgMap map[string]interface{} -} - -// Load load data from driver, for example load from database, -// it should be invoked before get any user scope config -// for system scope config, because it is immutable, no need to call this method -func (d *InMemoryDriver) Load() (map[string]interface{}, error) { - d.Lock() - defer d.Unlock() - res := make(map[string]interface{}) - for k, v := range d.cfgMap { - res[k] = v - } - return res, nil -} - -// Save only save user config setting to driver, for example: database, REST -func (d *InMemoryDriver) Save(cfg map[string]interface{}) error { - d.Lock() - defer d.Unlock() - for k, v := range cfg { - d.cfgMap[k] = v - } - return nil -} - -// NewInMemoryManager create a manager for unit testing, doesn't involve database or REST -func NewInMemoryManager() *CfgManager { - manager := &CfgManager{store: store.NewConfigStore(&InMemoryDriver{cfgMap: map[string]interface{}{}})} - // load default value - manager.loadDefault() - // load system config from env - manager.loadSystemConfigFromEnv() - return manager -} - -// loadDefault ... -func (c *CfgManager) loadDefault() { - // Init Default Value - itemArray := metadata.Instance().GetAll() - for _, item := range itemArray { - if len(item.DefaultValue) > 0 { - cfgValue, err := metadata.NewCfgValue(item.Name, item.DefaultValue) - if err != nil { - log.Errorf("loadDefault failed, config item, key: %v, err: %v", item.Name, err) - continue - } - c.store.Set(item.Name, *cfgValue) - } - } -} - -// loadSystemConfigFromEnv ... -func (c *CfgManager) loadSystemConfigFromEnv() { - itemArray := metadata.Instance().GetAll() - // Init System Value - for _, item := range itemArray { - if item.Scope == metadata.SystemScope && len(item.EnvKey) > 0 { - if envValue, ok := os.LookupEnv(item.EnvKey); ok { - configValue, err := metadata.NewCfgValue(item.Name, envValue) - if err != nil { - log.Errorf("loadSystemConfigFromEnv failed, config item, key: %v, err: %v", item.Name, err) - continue - } - c.store.Set(item.Name, *configValue) - } - } - } -} - -// GetAll get all settings. -func (c *CfgManager) GetAll() map[string]interface{} { - resultMap := map[string]interface{}{} - if err := c.store.Load(); err != nil { - log.Errorf("GetAll failed, error %v", err) - return resultMap - } - metaDataList := metadata.Instance().GetAll() - for _, item := range metaDataList { - cfgValue, err := c.store.GetAnyType(item.Name) - if err != nil { - if err != metadata.ErrValueNotSet { - log.Errorf("Failed to get value of key %v, error %v", item.Name, err) - } - continue - } - resultMap[item.Name] = cfgValue - } - return resultMap -} - -// GetUserCfgs retrieve all user configs -func (c *CfgManager) GetUserCfgs() map[string]interface{} { - resultMap := map[string]interface{}{} - if err := c.store.Load(); err != nil { - log.Errorf("GetUserCfgs failed, error %v", err) - return resultMap - } - metaDataList := metadata.Instance().GetAll() - for _, item := range metaDataList { - if item.Scope == metadata.UserScope { - cfgValue, err := c.store.GetAnyType(item.Name) - if err != nil { - if err == metadata.ErrValueNotSet { - if _, ok := item.ItemType.(*metadata.StringType); ok { - cfgValue = "" - } - if _, ok := item.ItemType.(*metadata.NonEmptyStringType); ok { - cfgValue = "" - } - } else { - log.Errorf("Failed to get value of key %v, error %v", item.Name, err) - continue - } - } - resultMap[item.Name] = cfgValue - } - } - return resultMap -} - -// Load load configuration from storage, like database or redis -func (c *CfgManager) Load() error { - return c.store.Load() -} - -// Save - Save all current configuration to storage -func (c *CfgManager) Save() error { - return c.store.Save() -} - -// Get ... -func (c *CfgManager) Get(key string) *metadata.ConfigureValue { - configValue, err := c.store.Get(key) - if err != nil { - log.Debugf("failed to get key %v, error: %v", key, err) - configValue = &metadata.ConfigureValue{} - } - return configValue -} - -// Set ... -func (c *CfgManager) Set(key string, value interface{}) { - configValue, err := metadata.NewCfgValue(key, utils.GetStrValueOfAnyType(value)) - if err != nil { - log.Errorf("error when setting key: %v, error %v", key, err) - return - } - c.store.Set(key, *configValue) -} - -// GetDatabaseCfg - Get database configurations -func (c *CfgManager) GetDatabaseCfg() *models.Database { - return &models.Database{ - Type: c.Get(common.DatabaseType).GetString(), - PostGreSQL: &models.PostGreSQL{ - Host: c.Get(common.PostGreSQLHOST).GetString(), - Port: c.Get(common.PostGreSQLPort).GetInt(), - Username: c.Get(common.PostGreSQLUsername).GetString(), - Password: c.Get(common.PostGreSQLPassword).GetString(), - Database: c.Get(common.PostGreSQLDatabase).GetString(), - SSLMode: c.Get(common.PostGreSQLSSLMode).GetString(), - MaxIdleConns: c.Get(common.PostGreSQLMaxIdleConns).GetInt(), - MaxOpenConns: c.Get(common.PostGreSQLMaxOpenConns).GetInt(), - }, - } -} - -// UpdateConfig - Update config store with a specified configuration and also save updated configure. -func (c *CfgManager) UpdateConfig(cfgs map[string]interface{}) error { - return c.store.Update(cfgs) -} - -// ValidateCfg validate config by metadata. return the first error if exist. -func (c *CfgManager) ValidateCfg(cfgs map[string]interface{}) error { - for key, value := range cfgs { - strVal := utils.GetStrValueOfAnyType(value) - _, err := metadata.NewCfgValue(key, strVal) - if err != nil { - return fmt.Errorf("%v, item name: %v", err, key) - } - } - return nil -} - -// DumpTrace dump all configurations -func (c *CfgManager) DumpTrace() { - cfgs := c.GetAll() - for k, v := range cfgs { - log.Info(k, ":=", v) - } -} diff --git a/src/common/config/manager_test.go b/src/common/config/manager_test.go deleted file mode 100644 index 00f9fa15e..000000000 --- a/src/common/config/manager_test.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "fmt" - "os" - "testing" - - "github.com/goharbor/harbor/src/common/utils/test" - "github.com/stretchr/testify/assert" -) - -var TestDBConfig = map[string]interface{}{ - "postgresql_host": "localhost", - "postgresql_database": "registry", - "postgresql_password": "root123", - "postgresql_username": "postgres", - "postgresql_sslmode": "disable", - "email_host": "127.0.0.1", - "scan_all_policy": `{"parameter":{"daily_time":0},"type":"daily"}`, -} - -var configManager *CfgManager - -func TestMain(m *testing.M) { - configManager = NewDBCfgManager() - test.InitDatabaseFromEnv() - configManager.UpdateConfig(TestDBConfig) - os.Exit(m.Run()) -} - -func TestLoadFromDatabase(t *testing.T) { - configManager.UpdateConfig(TestDBConfig) - configManager.Load() - assert.Equal(t, "127.0.0.1", configManager.Get("email_host").GetString()) - assert.Equal(t, `{"parameter":{"daily_time":0},"type":"daily"}`, configManager.Get("scan_all_policy").GetString()) -} - -func TestLoadUserCfg(t *testing.T) { - configMap := configManager.GetUserCfgs() - assert.NotNil(t, configMap["ldap_url"]) - assert.NotNil(t, configMap["ldap_base_dn"]) -} - -func TestSaveToDatabase(t *testing.T) { - fmt.Printf("database config %#v\n", configManager.GetDatabaseCfg()) - configManager.Load() - configManager.Set("read_only", "true") - configManager.Save() - configManager.Load() - assert.Equal(t, true, configManager.Get("read_only").GetBool()) -} - -func TestUpdateCfg(t *testing.T) { - testConfig := map[string]interface{}{ - "ldap_url": "ldaps://ldap.vmware.com", - "ldap_search_dn": "cn=admin,dc=example,dc=com", - "ldap_timeout": 10, - "ldap_search_password": "admin", - "ldap_base_dn": "dc=example,dc=com", - } - configManager.Load() - configManager.UpdateConfig(testConfig) - - assert.Equal(t, "ldaps://ldap.vmware.com", configManager.Get("ldap_url").GetString()) - assert.Equal(t, 10, configManager.Get("ldap_timeout").GetInt()) - assert.Equal(t, "admin", configManager.Get("ldap_search_password").GetPassword()) - assert.Equal(t, "cn=admin,dc=example,dc=com", configManager.Get("ldap_search_dn").GetString()) - assert.Equal(t, "dc=example,dc=com", configManager.Get("ldap_base_dn").GetString()) -} - -func TestCfgManager_loadDefaultValues(t *testing.T) { - configManager.loadDefault() - if configManager.Get("ldap_timeout").GetInt() != 5 { - t.Errorf("Failed to load ldap_timeout") - } -} - -func TestCfgManger_loadSystemValues(t *testing.T) { - configManager.loadDefault() - configManager.loadSystemConfigFromEnv() - configManager.UpdateConfig(map[string]interface{}{ - "postgresql_host": "127.0.0.1", - }) - if configManager.Get("postgresql_host").GetString() != "127.0.0.1" { - t.Errorf("Failed to set system value postgresql_host, expected %v, actual %v", "127.0.0.1", configManager.Get("postgresql_host").GetString()) - } -} -func TestCfgManager_GetDatabaseCfg(t *testing.T) { - configManager.UpdateConfig(map[string]interface{}{ - "postgresql_host": "localhost", - "postgresql_database": "registry", - "postgresql_password": "root123", - "postgresql_username": "postgres", - "postgresql_sslmode": "disable", - }) - dbCfg := configManager.GetDatabaseCfg() - assert.Equal(t, "localhost", dbCfg.PostGreSQL.Host) - assert.Equal(t, "registry", dbCfg.PostGreSQL.Database) - assert.Equal(t, "root123", dbCfg.PostGreSQL.Password) - assert.Equal(t, "postgres", dbCfg.PostGreSQL.Username) - assert.Equal(t, "disable", dbCfg.PostGreSQL.SSLMode) -} - -func TestNewInMemoryManager(t *testing.T) { - inMemoryManager := NewInMemoryManager() - inMemoryManager.UpdateConfig(map[string]interface{}{ - "ldap_url": "ldaps://ldap.vmware.com", - "ldap_timeout": 5, - "ldap_verify_cert": true, - }) - assert.Equal(t, "ldaps://ldap.vmware.com", inMemoryManager.Get("ldap_url").GetString()) - assert.Equal(t, 5, inMemoryManager.Get("ldap_timeout").GetInt()) - assert.Equal(t, true, inMemoryManager.Get("ldap_verify_cert").GetBool()) -} - -/* -func TestNewRESTCfgManager(t *testing.T) { - restMgr := NewRESTCfgManager("http://10.161.47.13:8080"+common.CoreConfigPath, "0XtgSGFx1amMDTaH") - err := restMgr.Load() - if err != nil { - t.Errorf("Failed with error %v", err) - } - fmt.Printf("db:%v", restMgr.GetDatabaseCfg().Type) - fmt.Printf("host:%#v\n", restMgr.GetDatabaseCfg().PostGreSQL.Host) - fmt.Printf("port:%#v\n", restMgr.GetDatabaseCfg().PostGreSQL.Port) - -}*/ diff --git a/src/common/config/store/driver/db_test.go b/src/common/config/store/driver/db_test.go deleted file mode 100644 index 97fe47bbc..000000000 --- a/src/common/config/store/driver/db_test.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package driver - -import ( - "fmt" - "os" - "testing" - - "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/config/metadata" - "github.com/goharbor/harbor/src/common/dao" - "github.com/stretchr/testify/assert" -) - -func TestMain(m *testing.M) { - dao.PrepareTestForPostgresSQL() - os.Exit(m.Run()) -} - -func TestDatabase_Load(t *testing.T) { - - cfgs := map[string]interface{}{ - common.AUTHMode: "db_auth", - common.LDAPURL: "ldap://ldap.vmware.com", - } - driver := Database{} - driver.Save(cfgs) - cfgMap, err := driver.Load() - if err != nil { - t.Errorf("failed to load, error %v", err) - } - assert.True(t, len(cfgMap) >= 1) - - if _, ok := cfgMap["ldap_url"]; !ok { - t.Error("Can not find ldap_url") - } -} - -func TestDatabase_Save(t *testing.T) { - ldapURL := "ldap://ldap.vmware.com" - driver := Database{} - prevCfg, err := driver.Load() - if err != nil { - t.Errorf("failed to load config %v", err) - } - cfgMap := map[string]interface{}{"ldap_url": ldapURL} - driver.Save(cfgMap) - updatedMap, err := driver.Load() - if err != nil { - t.Errorf("failed to load config %v", err) - } - assert.Equal(t, updatedMap["ldap_url"], ldapURL) - driver.Save(prevCfg) - -} - -func BenchmarkDatabaseLoad(b *testing.B) { - cfgs := map[string]interface{}{} - for _, item := range metadata.Instance().GetAll() { - cfgs[item.Name] = item.DefaultValue - } - - driver := Database{} - driver.Save(cfgs) - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - if _, err := driver.Load(); err != nil { - fmt.Printf("load failed %v", err) - } - } - }) -} diff --git a/src/common/config/store/driver/driver.go b/src/common/config/store/driver/driver.go deleted file mode 100644 index 505c77c6e..000000000 --- a/src/common/config/store/driver/driver.go +++ /dev/null @@ -1,10 +0,0 @@ -// Package driver provide the implementation of config driver used in CfgManager -package driver - -// Driver the interface to save/load config -type Driver interface { - // Load - load config item from config driver - Load() (map[string]interface{}, error) - // Save - save config item into config driver - Save(cfg map[string]interface{}) error -} diff --git a/src/common/config/store/driver/rest.go b/src/common/config/store/driver/rest.go deleted file mode 100644 index 607b932b6..000000000 --- a/src/common/config/store/driver/rest.go +++ /dev/null @@ -1,45 +0,0 @@ -package driver - -import ( - "errors" - "net/http" - - commonhttp "github.com/goharbor/harbor/src/common/http" - "github.com/goharbor/harbor/src/common/http/modifier" - "github.com/goharbor/harbor/src/lib/log" -) - -// RESTDriver - config store driver based on REST API -type RESTDriver struct { - configRESTURL string - client *commonhttp.Client -} - -// NewRESTDriver - Create RESTDriver -func NewRESTDriver(configRESTURL string, modifiers ...modifier.Modifier) *RESTDriver { - if commonhttp.InternalTLSEnabled() { - tr := commonhttp.GetHTTPTransport(commonhttp.SecureTransport) - return &RESTDriver{configRESTURL: configRESTURL, client: commonhttp.NewClient(&http.Client{Transport: tr}, modifiers...)} - - } - return &RESTDriver{configRESTURL: configRESTURL, client: commonhttp.NewClient(nil, modifiers...)} -} - -// Load - load config data from REST server -func (h *RESTDriver) Load() (map[string]interface{}, error) { - cfgMap := map[string]interface{}{} - log.Infof("get configuration from url: %+v", h.configRESTURL) - err := h.client.Get(h.configRESTURL, &cfgMap) - if err != nil { - log.Errorf("Failed on load rest config err:%v, url:%v", err, h.configRESTURL) - } - if len(cfgMap) < 1 { - return cfgMap, errors.New("failed to load rest config") - } - return cfgMap, err -} - -// Save - Save config data to REST server by PUT method -func (h *RESTDriver) Save(cfgMap map[string]interface{}) error { - return h.client.Put(h.configRESTURL, cfgMap) -} diff --git a/src/common/config/store/store_test.go b/src/common/config/store/store_test.go deleted file mode 100644 index 4fcf05fed..000000000 --- a/src/common/config/store/store_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package store - -import ( - "github.com/goharbor/harbor/src/common/config/metadata" - "github.com/goharbor/harbor/src/common/config/store/driver" - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/lib/log" - "github.com/stretchr/testify/assert" - "os" - "testing" -) - -func TestMain(m *testing.M) { - dao.PrepareTestForPostgresSQL() - cfgStore := NewConfigStore(&driver.Database{}) - cfgStore.Set("ldap_url", metadata.ConfigureValue{Name: "ldap_url", Value: "ldap://ldap.vmware.com"}) - err := cfgStore.Save() - if err != nil { - log.Fatal(err) - } - - os.Exit(m.Run()) -} -func TestConfigStore_Save(t *testing.T) { - cfgStore := NewConfigStore(&driver.Database{}) - err := cfgStore.Save() - cfgStore.Set("ldap_verify_cert", metadata.ConfigureValue{Name: "ldap_verify_cert", Value: "true"}) - if err != nil { - t.Fatal(err) - } - cfgValue, err := cfgStore.Get("ldap_verify_cert") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, true, cfgValue.GetBool()) - -} - -func TestConfigStore_Load(t *testing.T) { - cfgStore := NewConfigStore(&driver.Database{}) - err := cfgStore.Load() - if err != nil { - t.Fatal(err) - } - cfgValue, err := cfgStore.Get("ldap_url") - if err != nil { - t.Fatal(err) - } - - assert.Equal(t, "ldap://ldap.vmware.com", cfgValue.GetString()) - -} - -func TestToString(t *testing.T) { - cases := []struct { - name string - value interface{} - expect string - }{ - { - name: "transform int", - value: 999, - expect: "999", - }, - { - name: "transform slice", - value: []int{0, 1, 2}, - expect: "[0,1,2]", - }, - { - name: "transform map", - value: map[string]string{"k": "v"}, - expect: "{\"k\":\"v\"}", - }, - { - name: "transform bool", - value: false, - expect: "false", - }, - { - name: "transform nil", - value: nil, - expect: "nil", - }, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - s, err := toString(c.value) - assert.Nil(t, err) - assert.Equal(t, c.expect, s) - }) - } -} diff --git a/src/common/const.go b/src/common/const.go index 0d353c75d..e6a29ceb0 100755 --- a/src/common/const.go +++ b/src/common/const.go @@ -23,6 +23,9 @@ const ( UAAAuth = "uaa_auth" HTTPAuth = "http_auth" OIDCAuth = "oidc_auth" + DBCfgManager = "db_cfg_manager" + InMemoryCfgManager = "in_memory_manager" + RestCfgManager = "rest_config_manager" ProCrtRestrEveryone = "everyone" ProCrtRestrAdmOnly = "adminonly" LDAPScopeBase = 0 @@ -141,7 +144,7 @@ const ( RobotNamePrefix = "robot_name_prefix" // Use this prefix to index user who tries to login with web hook token. AuthProxyUserNamePrefix = "tokenreview$" - CoreConfigPath = "/api/internal/configurations" + CoreConfigPath = "/api/v2.0/internalconfig" RobotTokenDuration = "robot_token_duration" OIDCCallbackPath = "/c/oidc/callback" diff --git a/src/common/dao/base.go b/src/common/dao/base.go index eb08d7d56..06ee5bdcc 100644 --- a/src/common/dao/base.go +++ b/src/common/dao/base.go @@ -146,14 +146,6 @@ func PaginateForQuerySetter(qs orm.QuerySeter, page, size int64) orm.QuerySeter return qs } -// Escape .. -func Escape(str string) string { - str = strings.Replace(str, `\`, `\\`, -1) - str = strings.Replace(str, `%`, `\%`, -1) - str = strings.Replace(str, `_`, `\_`, -1) - return str -} - // implements github.com/golang-migrate/migrate/v4.Logger type mLogger struct { logger *log.Logger diff --git a/src/common/dao/config.go b/src/common/dao/config.go deleted file mode 100644 index 8f72a2dea..000000000 --- a/src/common/dao/config.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dao - -import ( - "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/utils" - "github.com/goharbor/harbor/src/lib/log" -) - -// AuthModeCanBeModified determines whether auth mode can be -// modified or not. Auth mode can modified when there is only admin -// user in database. -func AuthModeCanBeModified() (bool, error) { - c, err := GetOrmer().QueryTable(&models.User{}).Count() - if err != nil { - return false, err - } - // admin and anonymous - return c == 2, nil -} - -// GetConfigEntries Get configuration from database -func GetConfigEntries() ([]*models.ConfigEntry, error) { - o := GetOrmer() - var p []*models.ConfigEntry - sql := "select * from properties" - n, err := o.Raw(sql, []interface{}{}).QueryRows(&p) - - if err != nil { - return nil, err - } - - if n == 0 { - return nil, nil - } - return p, nil -} - -// SaveConfigEntries Save configuration to database. -func SaveConfigEntries(entries []models.ConfigEntry) error { - o := GetOrmer() - for _, entry := range entries { - if entry.Key == common.LDAPGroupAdminDn { - entry.Value = utils.TrimLower(entry.Value) - } - tempEntry := models.ConfigEntry{} - tempEntry.Key = entry.Key - tempEntry.Value = entry.Value - created, _, err := o.ReadOrCreate(&tempEntry, "k") - if err != nil && !IsDupRecErr(err) { - log.Errorf("Error create configuration entry: %v", err) - return err - } - if !created { - entry.ID = tempEntry.ID - _, err := o.Update(&entry, "v") - if err != nil { - return err - } - } - } - return nil -} diff --git a/src/common/dao/config_test.go b/src/common/dao/config_test.go index 5f82fc0a0..fea70c07c 100644 --- a/src/common/dao/config_test.go +++ b/src/common/dao/config_test.go @@ -15,19 +15,21 @@ package dao import ( + "github.com/goharbor/harbor/src/lib/orm" "testing" "github.com/goharbor/harbor/src/common/models" ) func TestAuthModeCanBeModified(t *testing.T) { + ctx := orm.Context() c, err := GetOrmer().QueryTable(&models.User{}).Count() if err != nil { t.Fatalf("failed to count users: %v", err) } if c == 2 { - flag, err := AuthModeCanBeModified() + flag, err := AuthModeCanBeModified(ctx) if err != nil { t.Fatalf("failed to determine whether auth mode can be modified: %v", err) } @@ -51,7 +53,7 @@ func TestAuthModeCanBeModified(t *testing.T) { } }(id) - flag, err = AuthModeCanBeModified() + flag, err = AuthModeCanBeModified(ctx) if err != nil { t.Fatalf("failed to determine whether auth mode can be modified: %v", err) } @@ -60,7 +62,7 @@ func TestAuthModeCanBeModified(t *testing.T) { } } else { - flag, err := AuthModeCanBeModified() + flag, err := AuthModeCanBeModified(ctx) if err != nil { t.Fatalf("failed to determine whether auth mode can be modified: %v", err) } diff --git a/src/common/dao/dao_test.go b/src/common/dao/dao_test.go index d7b10badd..938f8ddc7 100644 --- a/src/common/dao/dao_test.go +++ b/src/common/dao/dao_test.go @@ -15,20 +15,23 @@ package dao import ( + "context" "fmt" "os" "testing" "time" "github.com/astaxie/beego/orm" - "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/lib/log" + libOrm "github.com/goharbor/harbor/src/lib/orm" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +var testCtx context.Context + func execUpdate(o orm.Ormer, sql string, params ...interface{}) error { p, err := o.Raw(sql).Prepare() if err != nil { @@ -120,6 +123,7 @@ func TestMain(m *testing.M) { default: log.Fatalf("invalid database: %s", database) } + testCtx = libOrm.Context() result = testForAll(m) if result != 0 { @@ -566,104 +570,6 @@ func TestIsSuperUser(t *testing.T) { assert.False(IsSuperUser("none")) } -func TestSaveConfigEntries(t *testing.T) { - configEntries := []models.ConfigEntry{ - { - Key: "teststringkey", - Value: "192.168.111.211", - }, - { - Key: "testboolkey", - Value: "true", - }, - { - Key: "testnumberkey", - Value: "5", - }, - { - Key: common.CfgDriverDB, - Value: "db", - }, - } - err := SaveConfigEntries(configEntries) - if err != nil { - t.Fatalf("failed to save configuration to database %v", err) - } - readEntries, err := GetConfigEntries() - if err != nil { - t.Fatalf("Failed to get configuration from database %v", err) - } - findItem := 0 - for _, entry := range readEntries { - switch entry.Key { - case "teststringkey": - if "192.168.111.211" == entry.Value { - findItem++ - } - case "testnumberkey": - if "5" == entry.Value { - findItem++ - } - case "testboolkey": - if "true" == entry.Value { - findItem++ - } - default: - } - } - if findItem != 3 { - t.Fatalf("Should update 3 configuration but only update %d", findItem) - } - - configEntries = []models.ConfigEntry{ - { - Key: "teststringkey", - Value: "192.168.111.215", - }, - { - Key: "testboolkey", - Value: "false", - }, - { - Key: "testnumberkey", - Value: "7", - }, - { - Key: common.CfgDriverDB, - Value: "db", - }, - } - err = SaveConfigEntries(configEntries) - if err != nil { - t.Fatalf("failed to save configuration to database %v", err) - } - readEntries, err = GetConfigEntries() - if err != nil { - t.Fatalf("Failed to get configuration from database %v", err) - } - findItem = 0 - for _, entry := range readEntries { - switch entry.Key { - case "teststringkey": - if "192.168.111.215" == entry.Value { - findItem++ - } - case "testnumberkey": - if "7" == entry.Value { - findItem++ - } - case "testboolkey": - if "false" == entry.Value { - findItem++ - } - default: - } - } - if findItem != 3 { - t.Fatalf("Should update 3 configuration but only update %d", findItem) - } -} - func TestIsDupRecError(t *testing.T) { assert.True(t, IsDupRecErr(fmt.Errorf("pq: duplicate key value violates unique constraint \"properties_k_key\""))) assert.False(t, IsDupRecErr(fmt.Errorf("other error"))) diff --git a/src/common/dao/label.go b/src/common/dao/label.go index a3bfd18a4..a2c46f5bf 100644 --- a/src/common/dao/label.go +++ b/src/common/dao/label.go @@ -20,6 +20,7 @@ import ( "github.com/astaxie/beego/orm" "github.com/goharbor/harbor/src/common/models" + libOrm "github.com/goharbor/harbor/src/lib/orm" ) // AddLabel creates a label @@ -71,7 +72,7 @@ func getLabelQuerySetter(query *models.LabelQuery) orm.QuerySeter { qs := GetOrmer().QueryTable(&models.Label{}) if len(query.Name) > 0 { if query.FuzzyMatchName { - qs = qs.Filter("Name__icontains", Escape(query.Name)) + qs = qs.Filter("Name__icontains", libOrm.Escape(query.Name)) } else { qs = qs.Filter("Name", query.Name) } diff --git a/src/common/dao/pgsql.go b/src/common/dao/pgsql.go index a49d83314..283c4f125 100644 --- a/src/common/dao/pgsql.go +++ b/src/common/dao/pgsql.go @@ -16,12 +16,12 @@ package dao import ( "fmt" + "github.com/goharbor/harbor/src/common/models" "net/url" "os" "strconv" "github.com/astaxie/beego/orm" - "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/lib/log" migrate "github.com/golang-migrate/migrate/v4" diff --git a/src/common/dao/pro_meta.go b/src/common/dao/pro_meta.go index 25e412ba7..28e80a43b 100644 --- a/src/common/dao/pro_meta.go +++ b/src/common/dao/pro_meta.go @@ -16,7 +16,7 @@ package dao import ( "fmt" - "strings" + "github.com/goharbor/harbor/src/lib/orm" "time" "github.com/goharbor/harbor/src/common/models" @@ -43,7 +43,7 @@ func DeleteProjectMetadata(projectID int64, name ...string) error { params = append(params, projectID) if len(name) > 0 { - sql += fmt.Sprintf(` and name in ( %s )`, ParamPlaceholderForIn(len(name))) + sql += fmt.Sprintf(` and name in ( %s )`, orm.ParamPlaceholderForIn(len(name))) params = append(params, name) } @@ -73,7 +73,7 @@ func GetProjectMetadata(projectID int64, name ...string) ([]*models.ProjectMetad params = append(params, projectID) if len(name) > 0 { - sql += fmt.Sprintf(` and name in ( %s )`, ParamPlaceholderForIn(len(name))) + sql += fmt.Sprintf(` and name in ( %s )`, orm.ParamPlaceholderForIn(len(name))) params = append(params, name) } @@ -81,16 +81,6 @@ func GetProjectMetadata(projectID int64, name ...string) ([]*models.ProjectMetad return proMetas, err } -// ParamPlaceholderForIn returns a string that contains placeholders for sql keyword "in" -// e.g. n=3, returns "?,?,?" -func ParamPlaceholderForIn(n int) string { - placeholders := []string{} - for i := 0; i < n; i++ { - placeholders = append(placeholders, "?") - } - return strings.Join(placeholders, ",") -} - // ListProjectMetadata ... func ListProjectMetadata(name, value string) ([]*models.ProjectMetadata, error) { sql := `select * from project_metadata diff --git a/src/common/dao/project.go b/src/common/dao/project.go index 50b61d99f..adbed3ee4 100644 --- a/src/common/dao/project.go +++ b/src/common/dao/project.go @@ -17,7 +17,9 @@ package dao import ( "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/lib/log" + libOrm "github.com/goharbor/harbor/src/lib/orm" "fmt" "time" @@ -234,7 +236,7 @@ func projectQueryConditions(query *models.ProjectQueryParam) (string, []interfac if len(query.Name) != 0 { sql += ` and p.name like ?` - params = append(params, "%"+Escape(query.Name)+"%") + params = append(params, "%"+libOrm.Escape(query.Name)+"%") } if query.RegistryID > 0 { @@ -266,7 +268,7 @@ func projectQueryConditions(query *models.ProjectQueryParam) (string, []interfac } if len(query.ProjectIDs) > 0 { sql += fmt.Sprintf(` and p.project_id in ( %s )`, - ParamPlaceholderForIn(len(query.ProjectIDs))) + utils.ParamPlaceholderForIn(len(query.ProjectIDs))) params = append(params, query.ProjectIDs) } return sql, params diff --git a/src/common/dao/project/projectmember.go b/src/common/dao/project/projectmember.go index f44f450fe..5e31bc0b8 100644 --- a/src/common/dao/project/projectmember.go +++ b/src/common/dao/project/projectmember.go @@ -18,7 +18,9 @@ import ( "fmt" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/lib/orm" ) // GetProjectMember gets all members of the project. @@ -160,9 +162,9 @@ func SearchMemberByName(projectID int64, entityName string) ([]*models.Member, e order by entity_name ` queryParam := make([]interface{}, 4) queryParam = append(queryParam, projectID) - queryParam = append(queryParam, "%"+dao.Escape(entityName)+"%") + queryParam = append(queryParam, "%"+orm.Escape(entityName)+"%") queryParam = append(queryParam, projectID) - queryParam = append(queryParam, "%"+dao.Escape(entityName)+"%") + queryParam = append(queryParam, "%"+orm.Escape(entityName)+"%") members := []*models.Member{} log.Debugf("Query sql: %v", sql) _, err := o.Raw(sql, queryParam).QueryRows(&members) @@ -184,7 +186,7 @@ func ListRoles(user *models.User, projectID int64) ([]int, error) { sql += fmt.Sprintf(`union select role from project_member - where entity_type = 'g' and entity_id in ( %s ) and project_id = ? `, dao.ParamPlaceholderForIn(len(user.GroupIDs))) + where entity_type = 'g' and entity_id in ( %s ) and project_id = ? `, utils.ParamPlaceholderForIn(len(user.GroupIDs))) params = append(params, user.GroupIDs) params = append(params, projectID) } diff --git a/src/common/dao/project/projectmember_test.go b/src/common/dao/project/projectmember_test.go index 1d5649726..4f0a0e435 100644 --- a/src/common/dao/project/projectmember_test.go +++ b/src/common/dao/project/projectmember_test.go @@ -16,6 +16,7 @@ package project import ( "fmt" + "github.com/goharbor/harbor/src/controller/config" "os" "testing" @@ -28,7 +29,6 @@ import ( "github.com/goharbor/harbor/src/common/models" _ "github.com/goharbor/harbor/src/core/auth/db" _ "github.com/goharbor/harbor/src/core/auth/ldap" - cfg "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" ) @@ -71,7 +71,7 @@ func TestMain(m *testing.M) { "delete from project_member where id > 1", } dao.PrepareTestData(clearSqls, initSqls) - cfg.Init() + config.Init() result = m.Run() if result != 0 { diff --git a/src/common/dao/repository.go b/src/common/dao/repository.go index 326e07f45..8f6ab743e 100644 --- a/src/common/dao/repository.go +++ b/src/common/dao/repository.go @@ -16,6 +16,8 @@ package dao import ( "fmt" + "github.com/goharbor/harbor/src/common/utils" + libOrm "github.com/goharbor/harbor/src/lib/orm" "time" "github.com/astaxie/beego/orm" @@ -102,12 +104,12 @@ func repositoryQueryConditions(query ...*models.RepositoryQuery) (string, []inte if len(q.Name) > 0 { sql += `and r.name like ? ` - params = append(params, "%"+Escape(q.Name)+"%") + params = append(params, "%"+libOrm.Escape(q.Name)+"%") } if len(q.ProjectIDs) > 0 { sql += fmt.Sprintf(`and r.project_id in ( %s ) `, - ParamPlaceholderForIn(len(q.ProjectIDs))) + utils.ParamPlaceholderForIn(len(q.ProjectIDs))) params = append(params, q.ProjectIDs) } diff --git a/src/common/dao/testutils.go b/src/common/dao/testutils.go index a0d7101b0..63935c938 100644 --- a/src/common/dao/testutils.go +++ b/src/common/dao/testutils.go @@ -16,10 +16,10 @@ package dao import ( "fmt" + "github.com/goharbor/harbor/src/common/models" "os" "strconv" - "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/lib/log" ) diff --git a/src/common/dao/user.go b/src/common/dao/user.go index 9a9a4ba37..83ecd30ca 100644 --- a/src/common/dao/user.go +++ b/src/common/dao/user.go @@ -15,8 +15,10 @@ package dao import ( + "context" "errors" "fmt" + libOrm "github.com/goharbor/harbor/src/lib/orm" "time" "github.com/astaxie/beego/orm" @@ -129,11 +131,11 @@ func userQueryConditions(query *models.UserQuery) orm.QuerySeter { } if len(query.Username) > 0 { - qs = qs.Filter("username__contains", Escape(query.Username)) + qs = qs.Filter("username__contains", libOrm.Escape(query.Username)) } if len(query.Email) > 0 { - qs = qs.Filter("email__contains", Escape(query.Email)) + qs = qs.Filter("email__contains", libOrm.Escape(query.Email)) } return qs @@ -292,3 +294,19 @@ func CleanUser(id int64) error { func matchPassword(u *models.User, password string) bool { return utils.Encrypt(password, u.Salt, u.PasswordVersion) == u.Password } + +// AuthModeCanBeModified determines whether auth mode can be +// modified or not. Auth mode can modified when there is only admin +// user in database. +func AuthModeCanBeModified(ctx context.Context) (bool, error) { + o, err := libOrm.FromContext(ctx) + if err != nil { + return false, err + } + c, err := o.QueryTable(&models.User{}).Count() + if err != nil { + return false, err + } + // admin and anonymous + return c == 2, nil +} diff --git a/src/common/job/client.go b/src/common/job/client.go index 1030360b3..2d0e6c86e 100644 --- a/src/common/job/client.go +++ b/src/common/job/client.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/goharbor/harbor/src/controller/config" "io/ioutil" "net/http" "regexp" @@ -13,7 +14,6 @@ import ( commonhttp "github.com/goharbor/harbor/src/common/http" "github.com/goharbor/harbor/src/common/http/modifier/auth" "github.com/goharbor/harbor/src/common/job/models" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/jobservice/job" ) diff --git a/src/common/models/base.go b/src/common/models/base.go index 8cd6dff2a..d4ab0acf2 100644 --- a/src/common/models/base.go +++ b/src/common/models/base.go @@ -25,7 +25,6 @@ func init() { new(Role), new(RepoRecord), new(ProjectMetadata), - new(ConfigEntry), new(Label), new(ResourceLabel), new(UserGroup), diff --git a/src/common/models/database.go b/src/common/models/database.go new file mode 100644 index 000000000..3fb031240 --- /dev/null +++ b/src/common/models/database.go @@ -0,0 +1,47 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +// Database ... +type Database struct { + Type string `json:"type"` + PostGreSQL *PostGreSQL `json:"postgresql,omitempty"` +} + +// MySQL ... +type MySQL struct { + Host string `json:"host"` + Port int `json:"port"` + Username string `json:"username"` + Password string `json:"password,omitempty"` + Database string `json:"database"` +} + +// SQLite ... +type SQLite struct { + File string `json:"file"` +} + +// PostGreSQL ... +type PostGreSQL struct { + Host string `json:"host"` + Port int `json:"port"` + Username string `json:"username"` + Password string `json:"password,omitempty"` + Database string `json:"database"` + SSLMode string `json:"sslmode"` + MaxIdleConns int `json:"max_idle_conns"` + MaxOpenConns int `json:"max_open_conns"` +} diff --git a/src/common/utils/utils.go b/src/common/utils/utils.go index c7003651f..c23e62e46 100644 --- a/src/common/utils/utils.go +++ b/src/common/utils/utils.go @@ -295,3 +295,13 @@ func FindNamedMatches(regex *regexp.Regexp, str string) map[string]string { } return results } + +// ParamPlaceholderForIn returns a string that contains placeholders for sql keyword "in" +// e.g. n=3, returns "?,?,?" +func ParamPlaceholderForIn(n int) string { + placeholders := []string{} + for i := 0; i < n; i++ { + placeholders = append(placeholders, "?") + } + return strings.Join(placeholders, ",") +} diff --git a/src/controller/config/controller.go b/src/controller/config/controller.go new file mode 100644 index 000000000..c3884d643 --- /dev/null +++ b/src/controller/config/controller.go @@ -0,0 +1,217 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "context" + "encoding/json" + "fmt" + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/lib/config" + "github.com/goharbor/harbor/src/lib/config/metadata" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/config/db" + "github.com/goharbor/harbor/src/pkg/config/inmemory" +) + +var ( + // Ctl Global instance of the config controller + Ctl = NewController() +) + +// Controller define operations related to configures +type Controller interface { + // UserConfigs get the user scope configurations + UserConfigs(ctx context.Context) (map[string]*config.Value, error) + // UpdateUserConfigs update the user scope configurations + UpdateUserConfigs(ctx context.Context, conf map[string]interface{}) error + // GetAll get all configurations, used by internal, should include the system config items + AllConfigs(ctx context.Context) (map[string]interface{}, error) + // Load ... + Load(ctx context.Context) error + // GetString ... + GetString(ctx context.Context, item string) string + // GetBool ... + GetBool(ctx context.Context, item string) bool + // GetInt ... + GetInt(ctx context.Context, item string) int + // Get ... + Get(ctx context.Context, item string) *metadata.ConfigureValue + // GetCfgManager ... + GetManager() config.Manager +} + +type controller struct { + mgr config.Manager +} + +// NewController ... +func NewController() Controller { + return &controller{mgr: db.NewDBCfgManager()} +} + +// NewInMemoryController ... +func NewInMemoryController() Controller { + return &controller{mgr: inmemory.NewInMemoryManager()} +} + +func (c *controller) GetManager() config.Manager { + return c.mgr +} + +func (c *controller) Get(ctx context.Context, item string) *metadata.ConfigureValue { + return c.mgr.Get(ctx, item) +} + +func (c *controller) Load(ctx context.Context) error { + return c.mgr.Load(ctx) +} + +func (c *controller) GetString(ctx context.Context, item string) string { + return c.mgr.Get(ctx, item).GetString() +} + +func (c *controller) GetBool(ctx context.Context, item string) bool { + return c.mgr.Get(ctx, item).GetBool() +} + +func (c *controller) GetInt(ctx context.Context, item string) int { + return c.mgr.Get(ctx, item).GetInt() +} + +func (c *controller) UserConfigs(ctx context.Context) (map[string]*config.Value, error) { + configs := c.mgr.GetUserCfgs(ctx) + return ConvertForGet(ctx, configs, false) +} + +func (c *controller) AllConfigs(ctx context.Context) (map[string]interface{}, error) { + configs := c.mgr.GetAll(ctx) + return configs, nil +} + +func (c *controller) UpdateUserConfigs(ctx context.Context, conf map[string]interface{}) error { + err := c.mgr.Load(ctx) + if err != nil { + return err + } + isSysErr, err := c.validateCfg(ctx, conf) + if err != nil { + if isSysErr { + log.Errorf("failed to validate configurations: %v", err) + return fmt.Errorf("failed to validate configuration") + } + return err + } + if err := c.mgr.UpdateConfig(ctx, conf); err != nil { + log.Errorf("failed to upload configurations: %v", err) + return fmt.Errorf("failed to validate configuration") + } + return nil +} + +func (c *controller) validateCfg(ctx context.Context, cfgs map[string]interface{}) (bool, error) { + flag, err := authModeCanBeModified(ctx) + if err != nil { + return true, err + } + if !flag { + if failedKeys := c.checkUnmodifiable(ctx, cfgs, common.AUTHMode); len(failedKeys) > 0 { + return false, errors.BadRequestError(nil). + WithMessage(fmt.Sprintf("the keys %v can not be modified as new users have been inserted into database", failedKeys)) + } + } + err = c.mgr.ValidateCfg(ctx, cfgs) + if err != nil { + return false, errors.BadRequestError(err) + } + return false, nil +} + +func (c *controller) checkUnmodifiable(ctx context.Context, cfgs map[string]interface{}, keys ...string) (failed []string) { + if c.mgr == nil || cfgs == nil || keys == nil { + return + } + for _, k := range keys { + v := c.mgr.Get(ctx, k).GetString() + if nv, ok := cfgs[k]; ok { + if v != fmt.Sprintf("%v", nv) { + failed = append(failed, k) + } + } + } + return +} + +// ScanAllPolicy is represent the json request and object for scan all policy +// Only for migrating from the legacy schedule. +type ScanAllPolicy struct { + Type string `json:"type"` + Param map[string]interface{} `json:"parameter,omitempty"` +} + +// ConvertForGet - delete sensitive attrs and add editable field to every attr +func ConvertForGet(ctx context.Context, cfg map[string]interface{}, internal bool) (map[string]*config.Value, error) { + result := map[string]*config.Value{} + + mList := metadata.Instance().GetAll() + + for _, item := range mList { + val, exist := cfg[item.Name] + // skip undefined items + if !exist { + continue + } + + switch item.ItemType.(type) { + case *metadata.PasswordType: + // remove password for external api call + if !internal { + delete(cfg, item.Name) + continue + } + case *metadata.MapType, *metadata.StringToStringMapType: + // convert to string for map type + valByte, err := json.Marshal(val) + if err != nil { + return nil, err + } + val = string(valByte) + } + result[item.Name] = &config.Value{ + Val: val, + Editable: true, + } + } + + // default value for ScanAllPolicy + if _, ok := result[common.ScanAllPolicy]; !ok { + cfg[common.ScanAllPolicy] = `{"type":"none"}` + } + + // set value for auth_mode + flag, err := authModeCanBeModified(ctx) + if err != nil { + return nil, err + } + result[common.AUTHMode].Editable = flag + + return result, nil +} + +func authModeCanBeModified(ctx context.Context) (bool, error) { + return dao.AuthModeCanBeModified(ctx) +} diff --git a/src/controller/config/systemcfg.go b/src/controller/config/systemcfg.go new file mode 100644 index 000000000..a8b1b2af9 --- /dev/null +++ b/src/controller/config/systemcfg.go @@ -0,0 +1,250 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "context" + "errors" + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/secret" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/encrypt" + "os" + "strconv" + "strings" +) + +var ( + // Use backgroundCtx to access system scope config + backgroundCtx context.Context = context.Background() +) + +// It contains all system settings +// If the config is set in env, just get it from env +// If the config might not be set in env, and may have a default value, get it in this way: +// Ctl.GetString(backgroundCtx, "xxxx") + +// TokenPrivateKeyPath returns the path to the key for signing token for registry +func TokenPrivateKeyPath() string { + path := os.Getenv("TOKEN_PRIVATE_KEY_PATH") + if len(path) == 0 { + path = defaultRegistryTokenPrivateKeyPath + } + return path +} + +// RegistryURL ... +func RegistryURL() (string, error) { + url := os.Getenv("REGISTRY_URL") + if len(url) == 0 { + url = "http://registry:5000" + } + return url, nil +} + +// InternalJobServiceURL returns jobservice URL for internal communication between Harbor containers +func InternalJobServiceURL() string { + return os.Getenv("JOBSERVICE_URL") +} + +// GetCoreURL returns the url of core from env +func GetCoreURL() string { + return os.Getenv("CORE_URL") +} + +// CoreSecret returns a secret to mark harbor-core when communicate with +// other component +func CoreSecret() string { + return os.Getenv("CORE_SECRET") +} + +// RegistryCredential returns the username and password the core uses to access registry +func RegistryCredential() (string, string) { + return os.Getenv("REGISTRY_CREDENTIAL_USERNAME"), os.Getenv("REGISTRY_CREDENTIAL_PASSWORD") +} + +// JobserviceSecret returns a secret to mark Jobservice when communicate with +// other component +// TODO replace it with method of SecretStore +func JobserviceSecret() string { + return os.Getenv("JOBSERVICE_SECRET") +} + +// GetRedisOfRegURL returns the URL of Redis used by registry +func GetRedisOfRegURL() string { + return os.Getenv("_REDIS_URL_REG") +} + +// GetPortalURL returns the URL of portal +func GetPortalURL() string { + url := os.Getenv("PORTAL_URL") + if len(url) == 0 { + return common.DefaultPortalURL + } + return url +} + +// GetRegistryCtlURL returns the URL of registryctl +func GetRegistryCtlURL() string { + url := os.Getenv("REGISTRY_CONTROLLER_URL") + if len(url) == 0 { + return common.DefaultRegistryCtlURL + } + return url +} + +// GetPermittedRegistryTypesForProxyCache returns the permitted registry types for proxy cache +func GetPermittedRegistryTypesForProxyCache() []string { + types := os.Getenv("PERMITTED_REGISTRY_TYPES_FOR_PROXY_CACHE") + if len(types) == 0 { + return []string{} + } + return strings.Split(types, ",") +} + +// GetGCTimeWindow returns the reserve time window of blob. +func GetGCTimeWindow() int64 { + // the env is for testing/debugging. For production, Do NOT set it. + if env, exist := os.LookupEnv("GC_TIME_WINDOW_HOURS"); exist { + timeWindow, err := strconv.ParseInt(env, 10, 64) + if err == nil { + return timeWindow + } + } + return common.DefaultGCTimeWindowHours +} + +// WithNotary returns a bool value to indicate if Harbor's deployed with Notary +func WithNotary() bool { + return Ctl.GetBool(backgroundCtx, common.WithNotary) +} + +// WithTrivy returns a bool value to indicate if Harbor's deployed with Trivy. +func WithTrivy() bool { + return Ctl.GetBool(backgroundCtx, common.WithTrivy) +} + +// WithChartMuseum returns a bool to indicate if chartmuseum is deployed with Harbor. +func WithChartMuseum() bool { + return Ctl.GetBool(backgroundCtx, common.WithChartMuseum) +} + +// GetChartMuseumEndpoint returns the endpoint of the chartmuseum service +// otherwise an non nil error is returned +func GetChartMuseumEndpoint() (string, error) { + chartEndpoint := strings.TrimSpace(Ctl.GetString(backgroundCtx, common.ChartRepoURL)) + if len(chartEndpoint) == 0 { + return "", errors.New("empty chartmuseum endpoint") + } + return chartEndpoint, nil +} + +// ExtEndpoint returns the external URL of Harbor: protocol://host:port +func ExtEndpoint() (string, error) { + return Ctl.GetString(backgroundCtx, common.ExtEndpoint), nil +} + +// ExtURL returns the external URL: host:port +func ExtURL() (string, error) { + endpoint, err := ExtEndpoint() + if err != nil { + log.Errorf("failed to load config, error %v", err) + } + l := strings.Split(endpoint, "://") + if len(l) > 1 { + return l[1], nil + } + return endpoint, nil +} + +// SecretKey returns the secret key to encrypt the password of target +func SecretKey() (string, error) { + return keyProvider.Get(nil) +} + +func initKeyProvider() { + path := os.Getenv("KEY_PATH") + if len(path) == 0 { + path = defaultKeyPath + } + log.Infof("key path: %s", path) + keyProvider = encrypt.NewFileKeyProvider(path) +} + +func initSecretStore() { + m := map[string]string{} + m[JobserviceSecret()] = secret.JobserviceUser + SecretStore = secret.NewStore(m) +} + +// InternalCoreURL returns the local harbor core url +func InternalCoreURL() string { + return strings.TrimSuffix(Ctl.GetString(backgroundCtx, common.CoreURL), "/") +} + +// LocalCoreURL returns the local harbor core url +func LocalCoreURL() string { + return Ctl.GetString(backgroundCtx, common.CoreLocalURL) +} + +// InternalTokenServiceEndpoint returns token service endpoint for internal communication between Harbor containers +func InternalTokenServiceEndpoint() string { + return InternalCoreURL() + "/service/token" +} + +// InternalNotaryEndpoint returns notary server endpoint for internal communication between Harbor containers +// This is currently a conventional value and can be unaccessible when Harbor is not deployed with Notary. +func InternalNotaryEndpoint() string { + return Ctl.GetString(backgroundCtx, common.NotaryURL) +} + +// TrivyAdapterURL returns the endpoint URL of a Trivy adapter instance, by default it's the one deployed within Harbor. +func TrivyAdapterURL() string { + return Ctl.GetString(backgroundCtx, common.TrivyAdapterURL) +} + +// Metric returns the overall metric settings +func Metric() *models.Metric { + return &models.Metric{ + Enabled: Ctl.GetBool(backgroundCtx, common.MetricEnable), + Port: Ctl.GetInt(backgroundCtx, common.MetricPort), + Path: Ctl.GetString(backgroundCtx, common.MetricPath), + } +} + +// InitialAdminPassword returns the initial password for administrator +func InitialAdminPassword() (string, error) { + return Ctl.GetString(backgroundCtx, common.AdminInitialPassword), nil +} + +// Database returns database settings +func Database() (*models.Database, error) { + database := &models.Database{} + database.Type = Ctl.GetString(backgroundCtx, common.DatabaseType) + postgresql := &models.PostGreSQL{ + Host: Ctl.GetString(backgroundCtx, common.PostGreSQLHOST), + Port: Ctl.GetInt(backgroundCtx, common.PostGreSQLPort), + Username: Ctl.GetString(backgroundCtx, common.PostGreSQLUsername), + Password: Ctl.GetString(backgroundCtx, common.PostGreSQLPassword), + Database: Ctl.GetString(backgroundCtx, common.PostGreSQLDatabase), + SSLMode: Ctl.GetString(backgroundCtx, common.PostGreSQLSSLMode), + MaxIdleConns: Ctl.GetInt(backgroundCtx, common.PostGreSQLMaxIdleConns), + MaxOpenConns: Ctl.GetInt(backgroundCtx, common.PostGreSQLMaxOpenConns), + } + database.PostGreSQL = postgresql + + return database, nil +} diff --git a/src/core/config/test/ca.crt b/src/controller/config/test/ca.crt similarity index 100% rename from src/core/config/test/ca.crt rename to src/controller/config/test/ca.crt diff --git a/src/controller/config/test/controller_test.go b/src/controller/config/test/controller_test.go new file mode 100644 index 000000000..c7352c0ec --- /dev/null +++ b/src/controller/config/test/controller_test.go @@ -0,0 +1,145 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "context" + "github.com/goharbor/harbor/src/common" + . "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/errors" + htesting "github.com/goharbor/harbor/src/testing" + "github.com/stretchr/testify/suite" + "testing" +) + +var TestDBConfig = map[string]interface{}{ + common.LDAPBaseDN: "dc=example,dc=com", + common.LDAPURL: "ldap.example.com", + common.EmailHost: "127.0.0.1", +} + +var TestConfigWithScanAll = map[string]interface{}{ + "postgresql_host": "localhost", + "postgresql_database": "registry", + "postgresql_password": "root123", + "postgresql_username": "postgres", + "postgresql_sslmode": "disable", + "ldap_base_dn": "dc=example,dc=com", + "ldap_url": "ldap.example.com", + "email_host": "127.0.0.1", + "scan_all_policy": `{"parameter":{"daily_time":0},"type":"daily"}`, +} + +var ctx context.Context + +type controllerTestSuite struct { + htesting.Suite + controller Controller +} + +func (c *controllerTestSuite) SetupTest() { + c.controller = Ctl + ctx = c.Context() + c.controller.UpdateUserConfigs(ctx, TestDBConfig) +} + +func (c *controllerTestSuite) TestGetUserCfg() { + resp, err := c.controller.UserConfigs(ctx) + if err != nil { + c.Error(err, "failed to get user config") + } + c.Equal("dc=example,dc=com", resp["ldap_base_dn"].Val) + c.Equal("127.0.0.1", resp["email_host"].Val) + c.Equal("ldap.example.com", resp["ldap_url"].Val) +} + +func (c *controllerTestSuite) TestConvertForGet() { + conf := map[string]interface{}{ + "ldap_url": "ldaps.myexample,com", + "ldap_base_dn": "dc=myexample,dc=com", + "auth_mode": "ldap_auth", + "ldap_search_password": "admin", + } + + // password type should not sent to external api call + resp, err := ConvertForGet(ctx, conf, false) + c.Nil(err) + c.Equal("ldaps.myexample,com", resp["ldap_url"].Val) + c.Equal("ldap_auth", resp["auth_mode"].Val) + _, exist := resp["ldap_search_password"] + c.False(exist) + + // password type should be sent to internal api call + conf2 := map[string]interface{}{ + "ldap_url": "ldaps.myexample,com", + "ldap_base_dn": "dc=myexample,dc=com", + "auth_mode": "ldap_auth", + "ldap_search_password": "admin", + } + resp2, err2 := ConvertForGet(ctx, conf2, true) + c.Nil(err2) + c.Equal("ldaps.myexample,com", resp2["ldap_url"].Val) + c.Equal("ldap_auth", resp2["auth_mode"].Val) + _, exist2 := resp2["ldap_search_password"] + c.True(exist2) + +} + +func (c *controllerTestSuite) TestGetAll() { + resp, err := c.controller.AllConfigs(ctx) + if err != nil { + c.Error(err, "failed to get user config") + } + c.Equal("dc=example,dc=com", resp["ldap_base_dn"]) + c.Equal("127.0.0.1", resp["email_host"]) + c.Equal("ldap.example.com", resp["ldap_url"]) +} + +func (c *controllerTestSuite) TestUpdateUserCfg() { + + userConf := map[string]interface{}{ + common.LDAPURL: "ldaps.myexample,com", + common.LDAPBaseDN: "dc=myexample,dc=com", + } + err := c.controller.UpdateUserConfigs(ctx, userConf) + c.Nil(err) + cfgResp, err := c.controller.UserConfigs(ctx) + if err != nil { + c.Error(err, "failed to get user config") + } + c.Equal("dc=myexample,dc=com", cfgResp["ldap_base_dn"].Val) + c.Equal("ldaps.myexample,com", cfgResp["ldap_url"].Val) + badCfg := map[string]interface{}{ + common.LDAPScope: 5, + } + err2 := c.controller.UpdateUserConfigs(ctx, badCfg) + c.NotNil(err2) + c.True(errors.IsErr(err2, errors.BadRequestCode)) +} + +/*func (c *controllerTestSuite) TestCheckUnmodifiable() { + conf := map[string]interface{}{ + "ldap_url": "ldaps.myexample,com", + "ldap_base_dn": "dc=myexample,dc=com", + "auth_mode": "ldap_auth", + } + failed := c.controller.checkUnmodifiable(ctx, conf, "auth_mode") + c.True(len(failed) > 0) + c.Equal(failed[0], "auth_mode") +} +*/ +func TestControllerTestSuite(t *testing.T) { + suite.Run(t, &controllerTestSuite{}) +} diff --git a/src/controller/config/userconfig.go b/src/controller/config/userconfig.go new file mode 100755 index 000000000..5b3d72929 --- /dev/null +++ b/src/controller/config/userconfig.go @@ -0,0 +1,282 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package config provide config for core api and other modules +// Before accessing user settings, need to call Load() +// For system settings, no need to call Load() +package config + +import ( + "context" + "errors" + cfgModels "github.com/goharbor/harbor/src/lib/config/models" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/pkg/config" + "github.com/goharbor/harbor/src/pkg/encrypt" + "strings" + + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/secret" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/ldap/model" +) + +const ( + defaultKeyPath = "/etc/core/key" + defaultRegistryTokenPrivateKeyPath = "/etc/core/private_key.pem" + + // SessionCookieName is the name of the cookie for session ID + SessionCookieName = "sid" +) + +var ( + // SecretStore manages secrets + SecretStore *secret.Store + keyProvider encrypt.KeyProvider + // defined as a var for testing. + defaultCACertPath = "/etc/core/ca/ca.crt" +) + +// Init configurations +func Init() { + // init key provider + initKeyProvider() + log.Info("init secret store") + // init secret store + initSecretStore() +} + +// InitWithSettings init config with predefined configs, and optionally overwrite the keyprovider +func InitWithSettings(cfgs map[string]interface{}, kp ...encrypt.KeyProvider) { + Init() + Ctl = NewInMemoryController() + mgr := Ctl.GetManager() + mgr.UpdateConfig(backgroundCtx, cfgs) + if len(kp) > 0 { + keyProvider = kp[0] + } +} + +// GetCfgManager return the current config manager +func GetCfgManager(ctx context.Context) config.Manager { + return Ctl.GetManager() +} + +// Load configurations +func Load(ctx context.Context) error { + return Ctl.Load(ctx) +} + +// Upload save all configurations, used by testing +func Upload(cfg map[string]interface{}) error { + mgr := Ctl.GetManager() + return mgr.UpdateConfig(orm.Context(), cfg) +} + +// GetSystemCfg returns the system configurations +func GetSystemCfg(ctx context.Context) (map[string]interface{}, error) { + sysCfg, err := Ctl.AllConfigs(ctx) + if err != nil { + return nil, err + } + if len(sysCfg) == 0 { + return nil, errors.New("can not load system config, the database might be down") + } + return sysCfg, nil +} + +// AuthMode ... +func AuthMode(ctx context.Context) (string, error) { + err := Ctl.Load(ctx) + if err != nil { + log.Errorf("failed to load config, error %v", err) + return "db_auth", err + } + return Ctl.GetString(ctx, common.AUTHMode), nil +} + +// LDAPConf returns the setting of ldap server +func LDAPConf(ctx context.Context) (*model.LdapConf, error) { + err := Ctl.Load(ctx) + if err != nil { + return nil, err + } + return &model.LdapConf{ + URL: Ctl.GetString(ctx, common.LDAPURL), + SearchDn: Ctl.GetString(ctx, common.LDAPSearchDN), + SearchPassword: Ctl.GetString(ctx, common.LDAPSearchPwd), + BaseDn: Ctl.GetString(ctx, common.LDAPBaseDN), + UID: Ctl.GetString(ctx, common.LDAPUID), + Filter: Ctl.GetString(ctx, common.LDAPFilter), + Scope: Ctl.GetInt(ctx, common.LDAPScope), + ConnectionTimeout: Ctl.GetInt(ctx, common.LDAPTimeout), + VerifyCert: Ctl.GetBool(ctx, common.LDAPVerifyCert), + }, nil +} + +// LDAPGroupConf returns the setting of ldap group search +func LDAPGroupConf(ctx context.Context) (*model.GroupConf, error) { + err := Ctl.Load(ctx) + if err != nil { + return nil, err + } + return &model.GroupConf{ + BaseDN: Ctl.GetString(ctx, common.LDAPGroupBaseDN), + Filter: Ctl.GetString(ctx, common.LDAPGroupSearchFilter), + NameAttribute: Ctl.GetString(ctx, common.LDAPGroupAttributeName), + SearchScope: Ctl.GetInt(ctx, common.LDAPGroupSearchScope), + AdminDN: Ctl.GetString(ctx, common.LDAPGroupAdminDn), + MembershipAttribute: Ctl.GetString(ctx, common.LDAPGroupMembershipAttribute), + }, nil +} + +// TokenExpiration returns the token expiration time (in minute) +func TokenExpiration(ctx context.Context) (int, error) { + return Ctl.GetInt(ctx, common.TokenExpiration), nil +} + +// RobotTokenDuration returns the token expiration time of robot account (in minute) +func RobotTokenDuration(ctx context.Context) int { + return Ctl.GetInt(ctx, common.RobotTokenDuration) +} + +// SelfRegistration returns the enablement of self registration +func SelfRegistration(ctx context.Context) (bool, error) { + return Ctl.GetBool(ctx, common.SelfRegistration), nil +} + +// OnlyAdminCreateProject returns the flag to restrict that only sys admin can create project +func OnlyAdminCreateProject(ctx context.Context) (bool, error) { + err := Ctl.Load(ctx) + if err != nil { + return true, err + } + return Ctl.GetString(ctx, common.ProjectCreationRestriction) == common.ProCrtRestrAdmOnly, nil +} + +// Email returns email server settings +func Email(ctx context.Context) (*cfgModels.Email, error) { + err := Ctl.Load(ctx) + if err != nil { + return nil, err + } + return &cfgModels.Email{ + Host: Ctl.GetString(ctx, common.EmailHost), + Port: Ctl.GetInt(ctx, common.EmailPort), + Username: Ctl.GetString(ctx, common.EmailUsername), + Password: Ctl.GetString(ctx, common.EmailPassword), + SSL: Ctl.GetBool(ctx, common.EmailSSL), + From: Ctl.GetString(ctx, common.EmailFrom), + Identity: Ctl.GetString(ctx, common.EmailIdentity), + Insecure: Ctl.GetBool(ctx, common.EmailInsecure), + }, nil +} + +// UAASettings returns the UAASettings to access UAA service. +func UAASettings(ctx context.Context) (*models.UAASettings, error) { + err := Ctl.Load(ctx) + if err != nil { + return nil, err + } + us := &models.UAASettings{ + Endpoint: Ctl.GetString(ctx, common.UAAEndpoint), + ClientID: Ctl.GetString(ctx, common.UAAClientID), + ClientSecret: Ctl.GetString(ctx, common.UAAClientSecret), + VerifyCert: Ctl.GetBool(ctx, common.UAAVerifyCert), + } + return us, nil +} + +// ReadOnly returns a bool to indicates if Harbor is in read only mode. +func ReadOnly(ctx context.Context) bool { + return Ctl.GetBool(ctx, common.ReadOnly) +} + +// HTTPAuthProxySetting returns the setting of HTTP Auth proxy. the settings are only meaningful when the auth_mode is +// set to http_auth +func HTTPAuthProxySetting(ctx context.Context) (*cfgModels.HTTPAuthProxy, error) { + if err := Ctl.Load(ctx); err != nil { + return nil, err + } + return &cfgModels.HTTPAuthProxy{ + Endpoint: Ctl.GetString(ctx, common.HTTPAuthProxyEndpoint), + TokenReviewEndpoint: Ctl.GetString(ctx, common.HTTPAuthProxyTokenReviewEndpoint), + AdminGroups: splitAndTrim(Ctl.GetString(ctx, common.HTTPAuthProxyAdminGroups), ","), + AdminUsernames: splitAndTrim(Ctl.GetString(ctx, common.HTTPAuthProxyAdminUsernames), ","), + VerifyCert: Ctl.GetBool(ctx, common.HTTPAuthProxyVerifyCert), + SkipSearch: Ctl.GetBool(ctx, common.HTTPAuthProxySkipSearch), + ServerCertificate: Ctl.GetString(ctx, common.HTTPAuthProxyServerCertificate), + }, nil +} + +// OIDCSetting returns the setting of OIDC provider, currently there's only one OIDC provider allowed for Harbor and it's +// only effective when auth_mode is set to oidc_auth +func OIDCSetting(ctx context.Context) (*cfgModels.OIDCSetting, error) { + if err := Ctl.Load(ctx); err != nil { + return nil, err + } + scopeStr := Ctl.GetString(ctx, common.OIDCScope) + extEndpoint := strings.TrimSuffix(Ctl.GetString(nil, common.ExtEndpoint), "/") + scope := splitAndTrim(scopeStr, ",") + return &cfgModels.OIDCSetting{ + Name: Ctl.GetString(ctx, common.OIDCName), + Endpoint: Ctl.GetString(ctx, common.OIDCEndpoint), + VerifyCert: Ctl.GetBool(ctx, common.OIDCVerifyCert), + AutoOnboard: Ctl.GetBool(ctx, common.OIDCAutoOnboard), + ClientID: Ctl.GetString(ctx, common.OIDCCLientID), + ClientSecret: Ctl.GetString(ctx, common.OIDCClientSecret), + GroupsClaim: Ctl.GetString(ctx, common.OIDCGroupsClaim), + AdminGroup: Ctl.GetString(ctx, common.OIDCAdminGroup), + RedirectURL: extEndpoint + common.OIDCCallbackPath, + Scope: scope, + UserClaim: Ctl.GetString(ctx, common.OIDCUserClaim), + ExtraRedirectParms: Ctl.Get(ctx, common.OIDCExtraRedirectParms).GetStringToStringMap(), + }, nil +} + +// NotificationEnable returns a bool to indicates if notification enabled in harbor +func NotificationEnable(ctx context.Context) bool { + return Ctl.GetBool(ctx, common.NotificationEnable) +} + +// QuotaPerProjectEnable returns a bool to indicates if quota per project enabled in harbor +func QuotaPerProjectEnable(ctx context.Context) bool { + return Ctl.GetBool(ctx, common.QuotaPerProjectEnable) +} + +// QuotaSetting returns the setting of quota. +func QuotaSetting(ctx context.Context) (*cfgModels.QuotaSetting, error) { + if err := Ctl.Load(ctx); err != nil { + return nil, err + } + return &cfgModels.QuotaSetting{ + StoragePerProject: Ctl.Get(ctx, common.StoragePerProject).GetInt64(), + }, nil +} + +// RobotPrefix user defined robot name prefix. +func RobotPrefix(ctx context.Context) string { + return Ctl.GetString(ctx, common.RobotNamePrefix) +} + +func splitAndTrim(s, sep string) []string { + res := make([]string, 0) + for _, s := range strings.Split(s, sep) { + if e := strings.TrimSpace(s); len(e) > 0 { + res = append(res, e) + } + } + return res +} diff --git a/src/core/config/config_test.go b/src/controller/config/userconfig_test.go similarity index 85% rename from src/core/config/config_test.go rename to src/controller/config/userconfig_test.go index db090fcb0..da27ff8da 100644 --- a/src/core/config/config_test.go +++ b/src/controller/config/userconfig_test.go @@ -1,19 +1,21 @@ -// Copyright 2018 Project Harbor Authors +// Copyright Project Harbor Authors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package config import ( + "github.com/goharbor/harbor/src/lib/config/models" + "github.com/goharbor/harbor/src/lib/orm" "os" "path" "runtime" @@ -21,8 +23,8 @@ import ( "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/test" + _ "github.com/goharbor/harbor/src/pkg/config/inmemory" "github.com/stretchr/testify/assert" ) @@ -62,8 +64,8 @@ func TestConfig(t *testing.T) { os.Setenv("GC_TIME_WINDOW_HOURS", "0") Init() - - if err := Load(); err != nil { + ctx := orm.Context() + if err := Load(ctx); err != nil { t.Fatalf("failed to load configurations: %v", err) } @@ -71,11 +73,10 @@ func TestConfig(t *testing.T) { t.Fatalf("failed to upload configurations: %v", err) } - if _, err := GetSystemCfg(); err != nil { + if _, err := GetSystemCfg(ctx); err != nil { t.Fatalf("failed to get system configurations: %v", err) } - - mode, err := AuthMode() + mode, err := AuthMode(ctx) if err != nil { t.Fatalf("failed to get auth mode: %v", err) } @@ -83,19 +84,19 @@ func TestConfig(t *testing.T) { t.Errorf("unexpected mode: %s != %s", mode, "db_auth") } - if _, err := LDAPConf(); err != nil { + if _, err := LDAPConf(ctx); err != nil { t.Fatalf("failed to get ldap settings: %v", err) } - if _, err := LDAPGroupConf(); err != nil { + if _, err := LDAPGroupConf(ctx); err != nil { t.Fatalf("failed to get ldap group settings: %v", err) } - if _, err := TokenExpiration(); err != nil { + if _, err := TokenExpiration(ctx); err != nil { t.Fatalf("failed to get token expiration: %v", err) } - tkExp := RobotTokenDuration() + tkExp := RobotTokenDuration(ctx) assert.Equal(tkExp, 30) if _, err := ExtEndpoint(); err != nil { @@ -106,7 +107,7 @@ func TestConfig(t *testing.T) { t.Fatalf("failed to get secret key: %v", err) } - if _, err := SelfRegistration(); err != nil { + if _, err := SelfRegistration(ctx); err != nil { t.Fatalf("failed to get self registration: %v", err) } @@ -126,11 +127,11 @@ func TestConfig(t *testing.T) { t.Fatalf("failed to get initial admin password: %v", err) } - if _, err := OnlyAdminCreateProject(); err != nil { + if _, err := OnlyAdminCreateProject(ctx); err != nil { t.Fatalf("failed to get onldy admin create project: %v", err) } - if _, err := Email(); err != nil { + if _, err := Email(ctx); err != nil { t.Fatalf("failed to get email settings: %v", err) } @@ -150,7 +151,7 @@ func TestConfig(t *testing.T) { if WithTrivy() { t.Errorf("WithTrivy should be false") } - if ReadOnly() { + if ReadOnly(ctx) { t.Errorf("ReadOnly should be false") } @@ -162,7 +163,7 @@ func TestConfig(t *testing.T) { t.Errorf(`extURL should be "host01.com".`) } - mode, err = AuthMode() + mode, err = AuthMode(ctx) if err != nil { t.Fatalf("failed to get auth mode: %v", err) } @@ -174,7 +175,7 @@ func TestConfig(t *testing.T) { t.Errorf("Unexpected token private key path: %s, expected: %s", tokenKeyPath, "/etc/core/private_key.pem") } - us, err := UAASettings() + us, err := UAASettings(ctx) if err != nil { t.Fatalf("failed to get UAA setting, error: %v", err) } @@ -188,9 +189,9 @@ func TestConfig(t *testing.T) { localCoreURL := LocalCoreURL() assert.Equal("http://127.0.0.1:8080", localCoreURL) - assert.True(NotificationEnable()) + assert.True(NotificationEnable(ctx)) assert.Equal(int64(0), GetGCTimeWindow()) - assert.Equal("robot$", RobotPrefix()) + assert.Equal("robot$", RobotPrefix(ctx)) } @@ -243,7 +244,7 @@ y1bQusZMygQezfCuEzsewF+OpANFovCTUEs6s5vyoVNP8lk= common.HTTPAuthProxyServerCertificate: certificate, } InitWithSettings(m) - v, e := HTTPAuthProxySetting() + v, e := HTTPAuthProxySetting(orm.Context()) assert.Nil(t, e) assert.Equal(t, *v, models.HTTPAuthProxy{ Endpoint: "https://auth.proxy/suffix", @@ -269,7 +270,7 @@ func TestOIDCSetting(t *testing.T) { common.ExtEndpoint: "https://harbor.test", } InitWithSettings(m) - v, e := OIDCSetting() + v, e := OIDCSetting(orm.Context()) assert.Nil(t, e) assert.Equal(t, "test", v.Name) assert.Equal(t, "https://oidc.test", v.Endpoint) diff --git a/src/controller/event/handler/util/util.go b/src/controller/event/handler/util/util.go index 5a6cf005b..381c73b02 100644 --- a/src/controller/event/handler/util/util.go +++ b/src/controller/event/handler/util/util.go @@ -3,19 +3,22 @@ package util import ( "errors" "fmt" - "github.com/goharbor/harbor/src/core/config" + "strings" + + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/distribution" policy_model "github.com/goharbor/harbor/src/pkg/notification/policy/model" "github.com/goharbor/harbor/src/pkg/notifier/event" notifyModel "github.com/goharbor/harbor/src/pkg/notifier/model" - "strings" ) // SendHookWithPolicies send hook by publishing topic of specified target type(notify type) func SendHookWithPolicies(policies []*policy_model.Policy, payload *notifyModel.Payload, eventType string) error { // if global notification configured disabled, return directly - if !config.NotificationEnable() { + if !config.NotificationEnable(orm.Context()) { log.Debug("notification feature is not enabled") return nil } diff --git a/src/controller/event/handler/util/util_test.go b/src/controller/event/handler/util/util_test.go index c700fd1cc..96c519aec 100644 --- a/src/controller/event/handler/util/util_test.go +++ b/src/controller/event/handler/util/util_test.go @@ -1,12 +1,20 @@ package util import ( + "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/controller/config" + "os" "testing" "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/core/config" ) +func TestMain(m *testing.M) { + // do some initialization + test.InitDatabaseFromEnv() + os.Exit(m.Run()) +} + func TestBuildImageResourceURL(t *testing.T) { cfg := map[string]interface{}{ common.ExtEndpoint: "https://demo.goharbor.io", diff --git a/src/controller/event/handler/webhook/artifact/replication.go b/src/controller/event/handler/webhook/artifact/replication.go index 9df764853..467eb7bda 100644 --- a/src/controller/event/handler/webhook/artifact/replication.go +++ b/src/controller/event/handler/webhook/artifact/replication.go @@ -6,13 +6,14 @@ import ( "fmt" "strings" + "github.com/goharbor/harbor/src/controller/config" + commonModels "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/controller/event" "github.com/goharbor/harbor/src/controller/event/handler/util" ctlModel "github.com/goharbor/harbor/src/controller/event/model" "github.com/goharbor/harbor/src/controller/project" "github.com/goharbor/harbor/src/controller/replication" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/orm" @@ -33,7 +34,7 @@ func (r *ReplicationHandler) Name() string { // Handle ... func (r *ReplicationHandler) Handle(ctx context.Context, value interface{}) error { - if !config.NotificationEnable() { + if !config.NotificationEnable(ctx) { log.Debug("notification feature is not enabled") return nil } diff --git a/src/controller/event/handler/webhook/artifact/replication_test.go b/src/controller/event/handler/webhook/artifact/replication_test.go index fae3a565b..534776757 100644 --- a/src/controller/event/handler/webhook/artifact/replication_test.go +++ b/src/controller/event/handler/webhook/artifact/replication_test.go @@ -19,13 +19,14 @@ import ( "testing" "time" + "github.com/goharbor/harbor/src/controller/config" + common_dao "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/controller/event" "github.com/goharbor/harbor/src/controller/project" repctl "github.com/goharbor/harbor/src/controller/replication" repctlmodel "github.com/goharbor/harbor/src/controller/replication/model" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/pkg/notification" policy_model "github.com/goharbor/harbor/src/pkg/notification/policy/model" projecttesting "github.com/goharbor/harbor/src/testing/controller/project" diff --git a/src/controller/event/handler/webhook/artifact/retention.go b/src/controller/event/handler/webhook/artifact/retention.go index a8c73a0d1..f39f9b26f 100644 --- a/src/controller/event/handler/webhook/artifact/retention.go +++ b/src/controller/event/handler/webhook/artifact/retention.go @@ -3,14 +3,15 @@ package artifact import ( "context" "fmt" + "strings" + + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/controller/retention" "github.com/goharbor/harbor/src/lib/orm" - "strings" "github.com/goharbor/harbor/src/controller/event" "github.com/goharbor/harbor/src/controller/event/handler/util" evtModel "github.com/goharbor/harbor/src/controller/event/model" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/notification" @@ -28,7 +29,7 @@ func (r *RetentionHandler) Name() string { // Handle ... func (r *RetentionHandler) Handle(ctx context.Context, value interface{}) error { - if !config.NotificationEnable() { + if !config.NotificationEnable(ctx) { log.Debug("notification feature is not enabled") return nil } diff --git a/src/controller/event/handler/webhook/artifact/retention_test.go b/src/controller/event/handler/webhook/artifact/retention_test.go index dce5d8f78..a2168cf93 100644 --- a/src/controller/event/handler/webhook/artifact/retention_test.go +++ b/src/controller/event/handler/webhook/artifact/retention_test.go @@ -2,10 +2,16 @@ package artifact import ( "context" + "os" + "testing" + "time" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/controller/event" "github.com/goharbor/harbor/src/controller/retention" - "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/lib/selector" "github.com/goharbor/harbor/src/pkg/notification" policy_model "github.com/goharbor/harbor/src/pkg/notification/policy/model" @@ -15,9 +21,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "os" - "testing" - "time" ) func TestRetentionHandler_Handle(t *testing.T) { diff --git a/src/controller/event/handler/webhook/chart/chart.go b/src/controller/event/handler/webhook/chart/chart.go index 35e2597da..2d4886b92 100644 --- a/src/controller/event/handler/webhook/chart/chart.go +++ b/src/controller/event/handler/webhook/chart/chart.go @@ -18,12 +18,12 @@ import ( "context" "errors" "fmt" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/controller/event" "github.com/goharbor/harbor/src/controller/event/handler/util" "github.com/goharbor/harbor/src/controller/project" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/notifier/model" diff --git a/src/controller/event/handler/webhook/chart/chart_test.go b/src/controller/event/handler/webhook/chart/chart_test.go index a6f2edf38..f99fa5b26 100644 --- a/src/controller/event/handler/webhook/chart/chart_test.go +++ b/src/controller/event/handler/webhook/chart/chart_test.go @@ -16,12 +16,14 @@ package chart import ( "context" + testutils "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/controller/config" + "os" "testing" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/controller/event" "github.com/goharbor/harbor/src/controller/project" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/notification/policy/model" projecttesting "github.com/goharbor/harbor/src/testing/controller/project" @@ -31,6 +33,12 @@ import ( "github.com/stretchr/testify/require" ) +func TestMain(m *testing.M) { + // do some initialization + testutils.InitDatabaseFromEnv() + os.Exit(m.Run()) +} + func TestChartPreprocessHandler_Handle(t *testing.T) { PolicyMgr := notification.PolicyMgr defer func() { diff --git a/src/controller/event/handler/webhook/quota/quota_test.go b/src/controller/event/handler/webhook/quota/quota_test.go index 5cbee8e9f..b126a5345 100644 --- a/src/controller/event/handler/webhook/quota/quota_test.go +++ b/src/controller/event/handler/webhook/quota/quota_test.go @@ -17,6 +17,7 @@ package quota import ( "context" common_dao "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/controller/event" policy_model "github.com/goharbor/harbor/src/pkg/notification/policy/model" "github.com/goharbor/harbor/src/testing/mock" @@ -25,7 +26,6 @@ import ( "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/notification/policy" "github.com/goharbor/harbor/src/pkg/notifier" diff --git a/src/controller/event/handler/webhook/scan/scan_test.go b/src/controller/event/handler/webhook/scan/scan_test.go index edd3ab4c1..ecc5aee9b 100644 --- a/src/controller/event/handler/webhook/scan/scan_test.go +++ b/src/controller/event/handler/webhook/scan/scan_test.go @@ -17,6 +17,7 @@ package scan import ( "context" common_dao "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/controller/event" policy_model "github.com/goharbor/harbor/src/pkg/notification/policy/model" "testing" @@ -25,7 +26,6 @@ import ( "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/controller/artifact" sc "github.com/goharbor/harbor/src/controller/scan" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/notification/policy" "github.com/goharbor/harbor/src/pkg/notifier" diff --git a/src/controller/gc/callback.go b/src/controller/gc/callback.go index 82442b011..900792b66 100644 --- a/src/controller/gc/callback.go +++ b/src/controller/gc/callback.go @@ -4,9 +4,9 @@ import ( "context" "encoding/json" "fmt" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/controller/quota" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/orm" @@ -39,7 +39,7 @@ func gcCallback(ctx context.Context, p string) error { } func gcTaskStatusChange(ctx context.Context, taskID int64, status string) error { - if status == job.SuccessStatus.String() && config.QuotaPerProjectEnable() { + if status == job.SuccessStatus.String() && config.QuotaPerProjectEnable(ctx) { go func() { quota.RefreshForProjects(orm.Context()) }() diff --git a/src/controller/ldap/controller.go b/src/controller/ldap/controller.go index 584b1dcbb..58354059e 100644 --- a/src/controller/ldap/controller.go +++ b/src/controller/ldap/controller.go @@ -17,7 +17,7 @@ package ldap import ( "context" "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/ldap" "github.com/goharbor/harbor/src/pkg/ldap/model" @@ -52,7 +52,7 @@ func NewController() Controller { } func (c *controller) Session(ctx context.Context) (*ldap.Session, error) { - cfg, groupCfg, err := c.ldapConfigs() + cfg, groupCfg, err := c.ldapConfigs(ctx) if err != nil { return nil, err } @@ -61,7 +61,7 @@ func (c *controller) Session(ctx context.Context) (*ldap.Session, error) { func (c *controller) Ping(ctx context.Context, cfg model.LdapConf) (bool, error) { if len(cfg.SearchPassword) == 0 { - pwd, err := defaultPassword() + pwd, err := defaultPassword(ctx) if err != nil { return false, err } @@ -73,12 +73,12 @@ func (c *controller) Ping(ctx context.Context, cfg model.LdapConf) (bool, error) return c.mgr.Ping(ctx, cfg) } -func (c *controller) ldapConfigs() (*model.LdapConf, *model.GroupConf, error) { - cfg, err := config.LDAPConf() +func (c *controller) ldapConfigs(ctx context.Context) (*model.LdapConf, *model.GroupConf, error) { + cfg, err := config.LDAPConf(ctx) if err != nil { return nil, nil, err } - groupCfg, err := config.LDAPGroupConf() + groupCfg, err := config.LDAPGroupConf(ctx) if err != nil { log.Warningf("failed to get the ldap group config, error %v", err) groupCfg = &model.GroupConf{} @@ -87,20 +87,20 @@ func (c *controller) ldapConfigs() (*model.LdapConf, *model.GroupConf, error) { } func (c *controller) SearchUser(ctx context.Context, username string) ([]model.User, error) { - cfg, groupCfg, err := c.ldapConfigs() + cfg, groupCfg, err := c.ldapConfigs(ctx) if err != nil { return nil, err } return c.mgr.SearchUser(ctx, ldap.NewSession(*cfg, *groupCfg), username) } -func defaultPassword() (string, error) { - mod, err := config.AuthMode() +func defaultPassword(ctx context.Context) (string, error) { + mod, err := config.AuthMode(ctx) if err != nil { return "", err } if mod == common.LDAPAuth { - conf, err := config.LDAPConf() + conf, err := config.LDAPConf(ctx) if err != nil { return "", err } @@ -113,7 +113,7 @@ func defaultPassword() (string, error) { } func (c *controller) ImportUser(ctx context.Context, ldapImportUsers []string) ([]model.FailedImportUser, error) { - cfg, groupCfg, err := c.ldapConfigs() + cfg, groupCfg, err := c.ldapConfigs(ctx) if err != nil { return nil, err } @@ -121,7 +121,7 @@ func (c *controller) ImportUser(ctx context.Context, ldapImportUsers []string) ( } func (c *controller) SearchGroup(ctx context.Context, groupName, groupDN string) ([]model.Group, error) { - cfg, groupCfg, err := c.ldapConfigs() + cfg, groupCfg, err := c.ldapConfigs(ctx) if err != nil { return nil, err } diff --git a/src/controller/ldap/controller_test.go b/src/controller/ldap/controller_test.go index 36cf6181e..9c15e2238 100644 --- a/src/controller/ldap/controller_test.go +++ b/src/controller/ldap/controller_test.go @@ -16,7 +16,7 @@ package ldap import ( "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/pkg/ldap/model" htesting "github.com/goharbor/harbor/src/testing" "github.com/goharbor/harbor/src/testing/mock" diff --git a/src/controller/p2p/preheat/controllor_test.go b/src/controller/p2p/preheat/controllor_test.go index 4ea22eee0..81d4b344b 100644 --- a/src/controller/p2p/preheat/controllor_test.go +++ b/src/controller/p2p/preheat/controllor_test.go @@ -4,11 +4,11 @@ import ( "context" "errors" "fmt" + "github.com/goharbor/harbor/src/controller/config" "net/http" "net/http/httptest" "testing" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/orm" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/policy" diff --git a/src/controller/p2p/preheat/enforcer.go b/src/controller/p2p/preheat/enforcer.go index 5a761b5fc..ac0569396 100644 --- a/src/controller/p2p/preheat/enforcer.go +++ b/src/controller/p2p/preheat/enforcer.go @@ -17,6 +17,8 @@ package preheat import ( "context" "fmt" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" "strings" tk "github.com/docker/distribution/registry/auth/token" @@ -25,7 +27,6 @@ import ( "github.com/goharbor/harbor/src/controller/project" "github.com/goharbor/harbor/src/controller/scan" "github.com/goharbor/harbor/src/controller/tag" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/service/token" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib/errors" @@ -170,7 +171,7 @@ func NewEnforcer() Enforcer { Actions: []string{resourcePullAction}, }, } - t, err := token.MakeToken("distributor", token.Registry, ac) + t, err := token.MakeToken(orm.Context(), "distributor", token.Registry, ac) if err != nil { return "", err } diff --git a/src/controller/proxy/local.go b/src/controller/proxy/local.go index 5c35f978d..45333dbd9 100644 --- a/src/controller/proxy/local.go +++ b/src/controller/proxy/local.go @@ -18,8 +18,8 @@ import ( "context" "github.com/docker/distribution" "github.com/goharbor/harbor/src/controller/artifact" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/controller/event/metadata" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/cache" "github.com/goharbor/harbor/src/lib/errors" diff --git a/src/controller/quota/driver/project/project.go b/src/controller/quota/driver/project/project.go index 5c577390d..53d60dd86 100644 --- a/src/controller/quota/driver/project/project.go +++ b/src/controller/quota/driver/project/project.go @@ -17,10 +17,11 @@ package project import ( "context" "fmt" + "github.com/goharbor/harbor/src/lib/config" + "github.com/goharbor/harbor/src/pkg/config/db" "strconv" "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/config" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/controller/blob" "github.com/goharbor/harbor/src/lib/log" @@ -34,7 +35,7 @@ func init() { } type driver struct { - cfg *config.CfgManager + cfg config.Manager loader *dataloader.Loader blobCtl blob.Controller @@ -42,20 +43,20 @@ type driver struct { func (d *driver) Enabled(ctx context.Context, key string) (bool, error) { // NOTE: every time load the new configurations from the db to get the latest configurations may have performance problem. - if err := d.cfg.Load(); err != nil { + if err := d.cfg.Load(ctx); err != nil { return false, err } - return d.cfg.Get(common.QuotaPerProjectEnable).GetBool(), nil + return d.cfg.Get(ctx, common.QuotaPerProjectEnable).GetBool(), nil } func (d *driver) HardLimits(ctx context.Context) types.ResourceList { // NOTE: every time load the new configurations from the db to get the latest configurations may have performance problem. - if err := d.cfg.Load(); err != nil { + if err := d.cfg.Load(ctx); err != nil { log.Warningf("load configurations failed, error: %v", err) } return types.ResourceList{ - types.ResourceStorage: d.cfg.Get(common.StoragePerProject).GetInt64(), + types.ResourceStorage: d.cfg.Get(ctx, common.StoragePerProject).GetInt64(), } } @@ -118,7 +119,7 @@ func (d *driver) CalculateUsage(ctx context.Context, key string) (types.Resource } func newDriver() dr.Driver { - cfg := config.NewDBCfgManager() + cfg := db.NewDBCfgManager() loader := dataloader.NewBatchedLoader(getProjectsBatchFn, dataloader.WithClearCacheOnBatch()) diff --git a/src/controller/robot/controller.go b/src/controller/robot/controller.go index 03cc18f58..533eb06d7 100644 --- a/src/controller/robot/controller.go +++ b/src/controller/robot/controller.go @@ -5,7 +5,7 @@ import ( "fmt" rbac_project "github.com/goharbor/harbor/src/common/rbac/project" "github.com/goharbor/harbor/src/common/utils" - "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/q" @@ -85,8 +85,8 @@ func (d *controller) Create(ctx context.Context, r *Robot) (int64, string, error expiresAt = -1 } else if r.Duration == 0 { // system default robot duration - r.Duration = int64(config.RobotTokenDuration()) - expiresAt = time.Now().AddDate(0, 0, config.RobotTokenDuration()).Unix() + r.Duration = int64(config.RobotTokenDuration(ctx)) + expiresAt = time.Now().AddDate(0, 0, config.RobotTokenDuration(ctx)).Unix() } else { expiresAt = time.Now().AddDate(0, 0, int(r.Duration)).Unix() } @@ -216,7 +216,7 @@ func (d *controller) populate(ctx context.Context, r *model.Robot, option *Optio // for the v2 robots, add prefix to the robot name // for the v1 legacy robots, keep the robot name if robot.Editable { - robot.Name = fmt.Sprintf("%s%s", config.RobotPrefix(), r.Name) + robot.Name = fmt.Sprintf("%s%s", config.RobotPrefix(ctx), r.Name) } else { robot.Name = r.Name } diff --git a/src/controller/robot/controller_test.go b/src/controller/robot/controller_test.go index f7912dbaf..98429c2a2 100644 --- a/src/controller/robot/controller_test.go +++ b/src/controller/robot/controller_test.go @@ -6,12 +6,12 @@ import ( "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/test" - "github.com/goharbor/harbor/src/core/config" - core_cfg "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/permission/types" rbac_model "github.com/goharbor/harbor/src/pkg/rbac/model" "github.com/goharbor/harbor/src/pkg/robot/model" + htesting "github.com/goharbor/harbor/src/testing" "github.com/goharbor/harbor/src/testing/mock" "github.com/goharbor/harbor/src/testing/pkg/project" "github.com/goharbor/harbor/src/testing/pkg/rbac" @@ -22,7 +22,7 @@ import ( ) type ControllerTestSuite struct { - suite.Suite + htesting.Suite } func (suite *ControllerTestSuite) TestGet() { @@ -92,7 +92,7 @@ func (suite *ControllerTestSuite) TestCreate() { conf := map[string]interface{}{ common.RobotTokenDuration: "30", } - core_cfg.InitWithSettings(conf) + config.InitWithSettings(conf) projectMgr := &project.Manager{} rbacMgr := &rbac.Manager{} diff --git a/src/controller/robot/model_test.go b/src/controller/robot/model_test.go index acf55758e..6ebdeab6b 100644 --- a/src/controller/robot/model_test.go +++ b/src/controller/robot/model_test.go @@ -3,12 +3,13 @@ package robot import ( "github.com/goharbor/harbor/src/pkg/permission/types" "github.com/goharbor/harbor/src/pkg/robot/model" + htesting "github.com/goharbor/harbor/src/testing" "github.com/stretchr/testify/suite" "testing" ) type ModelTestSuite struct { - suite.Suite + htesting.Suite } func (suite *ModelTestSuite) TestSetLevel() { diff --git a/src/controller/scan/base_controller.go b/src/controller/scan/base_controller.go index de54f7f83..11b7ff73a 100644 --- a/src/controller/scan/base_controller.go +++ b/src/controller/scan/base_controller.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "fmt" + "github.com/goharbor/harbor/src/controller/config" "strings" "sync" @@ -25,7 +26,6 @@ import ( ar "github.com/goharbor/harbor/src/controller/artifact" "github.com/goharbor/harbor/src/controller/robot" sc "github.com/goharbor/harbor/src/controller/scanner" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/errors" diff --git a/src/controller/scan/base_controller_test.go b/src/controller/scan/base_controller_test.go index 9bab8a8e6..eff1096dc 100644 --- a/src/controller/scan/base_controller_test.go +++ b/src/controller/scan/base_controller_test.go @@ -19,6 +19,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "github.com/goharbor/harbor/src/controller/config" "testing" "time" @@ -26,7 +27,6 @@ import ( "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/controller/artifact" "github.com/goharbor/harbor/src/controller/robot" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/orm" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/permission/types" diff --git a/src/controller/systeminfo/controller.go b/src/controller/systeminfo/controller.go index 9e21952df..f0e749719 100644 --- a/src/controller/systeminfo/controller.go +++ b/src/controller/systeminfo/controller.go @@ -17,14 +17,14 @@ package systeminfo import ( "context" "fmt" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/config/models" "io" "os" "strings" "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/systeminfo" @@ -84,7 +84,7 @@ type controller struct{} func (c *controller) GetInfo(ctx context.Context, opt Options) (*Data, error) { logger := log.GetLogger(ctx) - cfg, err := config.GetSystemCfg() + cfg, err := config.GetSystemCfg(ctx) if err != nil { logger.Errorf("Error occurred getting config: %v", err) return nil, err @@ -95,7 +95,7 @@ func (c *controller) GetInfo(ctx context.Context, opt Options) (*Data, error) { HarborVersion: fmt.Sprintf("%s-%s", version.ReleaseVersion, version.GitCommit), } if res.AuthMode == common.HTTPAuth { - if s, err := config.HTTPAuthProxySetting(); err == nil { + if s, err := config.HTTPAuthProxySetting(ctx); err == nil { res.AuthProxySettings = s } else { logger.Warningf("Failed to get auth proxy setting, error: %v", err) @@ -117,7 +117,7 @@ func (c *controller) GetInfo(ctx context.Context, opt Options) (*Data, error) { res.Protected = &protectedData{ WithNotary: config.WithNotary(), WithChartMuseum: config.WithChartMuseum(), - ReadOnly: config.ReadOnly(), + ReadOnly: config.ReadOnly(ctx), ExtURL: extURL, RegistryURL: registryURL, HasCARoot: enableCADownload, diff --git a/src/controller/systeminfo/controller_test.go b/src/controller/systeminfo/controller_test.go index 61fdb8d68..983cd9000 100644 --- a/src/controller/systeminfo/controller_test.go +++ b/src/controller/systeminfo/controller_test.go @@ -2,17 +2,18 @@ package systeminfo import ( "context" + "github.com/goharbor/harbor/src/controller/config" + htesting "github.com/goharbor/harbor/src/testing" "testing" "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/pkg/version" "github.com/stretchr/testify/suite" ) type sysInfoCtlTestSuite struct { - suite.Suite + htesting.Suite ctl Controller } diff --git a/src/controller/tag/controller_test.go b/src/controller/tag/controller_test.go index 1dd1b29cb..d0b917457 100644 --- a/src/controller/tag/controller_test.go +++ b/src/controller/tag/controller_test.go @@ -16,7 +16,7 @@ package tag import ( "github.com/goharbor/harbor/src/common" - coreConfig "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/orm" pkg_artifact "github.com/goharbor/harbor/src/pkg/artifact" @@ -54,7 +54,7 @@ func (c *controllerTestSuite) SetupTest() { var tagCtlTestConfig = map[string]interface{}{ common.WithNotary: false, } - coreConfig.InitWithSettings(tagCtlTestConfig) + config.InitWithSettings(tagCtlTestConfig) } func (c *controllerTestSuite) TestEnsureTag() { diff --git a/src/core/api/base.go b/src/core/api/base.go index 974590f52..90e158a4d 100644 --- a/src/core/api/base.go +++ b/src/core/api/base.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/goharbor/harbor/src/controller/config" "net/http" "github.com/ghodss/yaml" @@ -29,7 +30,6 @@ import ( "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/controller/p2p/preheat" projectcontroller "github.com/goharbor/harbor/src/controller/project" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/scheduler" diff --git a/src/core/api/chart_repository.go b/src/core/api/chart_repository.go index 12dae28e6..75e1337ff 100755 --- a/src/core/api/chart_repository.go +++ b/src/core/api/chart_repository.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "github.com/goharbor/harbor/src/controller/config" "io" "io/ioutil" "mime/multipart" @@ -21,7 +22,6 @@ import ( rep_event "github.com/goharbor/harbor/src/controller/event/handler/replication/event" "github.com/goharbor/harbor/src/controller/event/metadata" "github.com/goharbor/harbor/src/controller/project" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/label" hlog "github.com/goharbor/harbor/src/lib/log" n_event "github.com/goharbor/harbor/src/pkg/notifier/event" diff --git a/src/core/api/config.go b/src/core/api/config.go deleted file mode 100644 index b16a7a124..000000000 --- a/src/core/api/config.go +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "errors" - "fmt" - "strings" - - "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/config" - "github.com/goharbor/harbor/src/common/config/metadata" - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/security" - corecfg "github.com/goharbor/harbor/src/core/config" - "github.com/goharbor/harbor/src/lib/log" -) - -// ConfigAPI ... -type ConfigAPI struct { - BaseController - cfgManager *config.CfgManager -} - -// Prepare validates the user -func (c *ConfigAPI) Prepare() { - c.BaseController.Prepare() - c.cfgManager = corecfg.GetCfgManager() - if !c.SecurityCtx.IsAuthenticated() { - c.SendUnAuthorizedError(errors.New("UnAuthorized")) - return - } - - // Only internal container can access /api/internal/configurations - if strings.EqualFold(c.Ctx.Request.RequestURI, "/api/internal/configurations") { - if s, ok := security.FromContext(c.Ctx.Request.Context()); !ok || s.Name() != "secret" { - c.SendUnAuthorizedError(errors.New("UnAuthorized")) - return - } - } - - if !c.SecurityCtx.IsSysAdmin() && !c.SecurityCtx.IsSolutionUser() { - c.SendForbiddenError(errors.New(c.SecurityCtx.GetUsername())) - return - } - -} - -type value struct { - Value interface{} `json:"value"` - Editable bool `json:"editable"` -} - -// Get returns configurations -func (c *ConfigAPI) Get() { - configs := c.cfgManager.GetUserCfgs() - m, err := convertForGet(configs) - if err != nil { - log.Errorf("failed to convert configurations: %v", err) - c.SendInternalServerError(errors.New("")) - return - } - - c.Data["json"] = m - c.ServeJSON() -} - -// GetInternalConfig returns internal configurations -func (c *ConfigAPI) GetInternalConfig() { - - configs := c.cfgManager.GetAll() - c.Data["json"] = configs - c.ServeJSON() -} - -// Put updates configurations -func (c *ConfigAPI) Put() { - m := map[string]interface{}{} - if err := c.DecodeJSONReq(&m); err != nil { - c.SendBadRequestError(err) - return - } - err := c.cfgManager.Load() - if err != nil { - log.Errorf("failed to get configurations: %v", err) - c.SendInternalServerError(errors.New("")) - return - } - isSysErr, err := c.validateCfg(m) - if err != nil { - if isSysErr { - log.Errorf("failed to validate configurations: %v", err) - c.SendInternalServerError(errors.New("")) - return - } - - c.SendBadRequestError(err) - return - - } - - if err := c.cfgManager.UpdateConfig(m); err != nil { - log.Errorf("failed to upload configurations: %v", err) - c.SendInternalServerError(errors.New("")) - return - } -} - -func (c *ConfigAPI) validateCfg(cfgs map[string]interface{}) (bool, error) { - flag, err := authModeCanBeModified() - if err != nil { - return true, err - } - if !flag { - if failedKeys := checkUnmodifiable(c.cfgManager, cfgs, common.AUTHMode); len(failedKeys) > 0 { - return false, fmt.Errorf("the keys %v can not be modified as new users have been inserted into database", failedKeys) - } - } - err = c.cfgManager.ValidateCfg(cfgs) - return false, err -} - -func checkUnmodifiable(mgr *config.CfgManager, cfgs map[string]interface{}, keys ...string) (failed []string) { - if mgr == nil || cfgs == nil || keys == nil { - return - } - for _, k := range keys { - v := mgr.Get(k).GetString() - if nv, ok := cfgs[k]; ok { - if v != fmt.Sprintf("%v", nv) { - failed = append(failed, k) - } - } - } - return -} - -// ScanAllPolicy is represent the json request and object for scan all policy -// Only for migrating from the legacy schedule. -type ScanAllPolicy struct { - Type string `json:"type"` - Param map[string]interface{} `json:"parameter,omitempty"` -} - -// delete sensitive attrs and add editable field to every attr -func convertForGet(cfg map[string]interface{}) (map[string]*value, error) { - result := map[string]*value{} - - mList := metadata.Instance().GetAll() - - for _, item := range mList { - if _, ok := item.ItemType.(*metadata.PasswordType); ok { - delete(cfg, item.Name) - } - } - - if _, ok := cfg[common.ScanAllPolicy]; !ok { - cfg[common.ScanAllPolicy] = ScanAllPolicy{ - Type: "none", // For legacy compatible - } - } - for k, v := range cfg { - result[k] = &value{ - Value: v, - Editable: true, - } - } - - flag, err := authModeCanBeModified() - if err != nil { - return nil, err - } - result[common.AUTHMode].Editable = flag - return result, nil -} - -func authModeCanBeModified() (bool, error) { - return dao.AuthModeCanBeModified() -} diff --git a/src/core/api/config_test.go b/src/core/api/config_test.go deleted file mode 100644 index 42c764675..000000000 --- a/src/core/api/config_test.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "fmt" - "testing" - - "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/core/config" - "github.com/stretchr/testify/assert" -) - -func TestGetConfig(t *testing.T) { - fmt.Println("Testing getting configurations") - assert := assert.New(t) - apiTest := newHarborAPI() - - // case 1: get configurations without admin role - code, _, err := apiTest.GetConfig(*testUser) - if err != nil { - t.Fatalf("failed to get configurations: %v", err) - } - - assert.Equal(401, code, "the status code of getting configurations with non-admin user should be 401") - - // case 2: get configurations with admin role - code, cfg, err := apiTest.GetConfig(*admin) - if err != nil { - t.Fatalf("failed to get configurations: %v", err) - } - - if !assert.Equal(200, code, "the status code of getting configurations with admin user should be 200") { - return - } - t.Logf("cfg: %+v", cfg) - mode := cfg[common.AUTHMode].Value.(string) - assert.Equal(common.DBAuth, mode, fmt.Sprintf("the auth mode should be %s", common.DBAuth)) - ccc, err := config.GetSystemCfg() - if err != nil { - t.Logf("failed to get system configurations: %v", err) - } - t.Logf("%v", ccc) -} - -func TestInternalConfig(t *testing.T) { - fmt.Println("Testing internal configurations") - assert := assert.New(t) - apiTest := newHarborAPI() - - // case 1: get configurations without admin role - code, _, err := apiTest.GetInternalConfig(*testUser) - if err != nil { - t.Fatalf("failed to get configurations: %v", err) - } - - assert.Equal(401, code, "the status code of getting configurations with non-admin user should be 401") - - // case 2: get configurations with admin role - code, _, err = apiTest.GetInternalConfig(*admin) - if err != nil { - t.Fatalf("failed to get configurations: %v", err) - } - - if !assert.Equal(200, code, "the status code of getting configurations with admin user should be 200") { - return - } -} - -func TestPutConfig(t *testing.T) { - fmt.Println("Testing modifying configurations") - assert := assert.New(t) - apiTest := newHarborAPI() - - cfg := map[string]interface{}{ - common.TokenExpiration: 60, - } - - code, err := apiTest.PutConfig(*admin, cfg) - if err != nil { - t.Fatalf("failed to get configurations: %v", err) - } - - if !assert.Equal(200, code, "the status code of modifying configurations with admin user should be 200") { - return - } - ccc, err := config.GetSystemCfg() - if err != nil { - t.Logf("failed to get system configurations: %v", err) - } - t.Logf("%v", ccc) -} - -func TestPutConfigMaxLength(t *testing.T) { - fmt.Println("Testing modifying configurations with max length.") - assert := assert.New(t) - apiTest := newHarborAPI() - - // length is 1059，expected code: 200 - cfg := map[string]interface{}{ - common.LDAPGroupSearchFilter: "YU2YcM13JtSx5jtBiftTjfaEM9KZFQ0XA5fKQHU02E9Xe0aLYaSy7YBokrTA8oHFjSkWFSgWZJ6FEmTS" + - "Vy5Ovsy5to2kWnFtbVNX3pzbeQpZeAqK3mEGnXdMkMSQu9WTq74s99GpwjEdA628pcZqLx6wCR0IvwryqIcNoRtqPlUcuRGODWA8ZXaC0d" + - "Qs7cRUYSe8onHsM2c9JWuUS8Jv4E7KggfytrxeKAT0WGP5DBZsB7rHZKxoAppE3C0NueEeC4yV791PUOODJt9rc0RrcD6ORUIO5RriCwym" + - "IinJZa03MtTk3vGFTmL9wM0wEYZP3fEBmoiB0iF8o4wkHGyMpNJoDyPuo7huuCbipAXClEcX1R7xD4aijTF9iOMKymvsObMZ4qqI7flco5" + - "yLFf7W8cpSisk3YJSvxDWfrl91WT4IFE5KHK976DgLQJhTZ8msGOImnFiUGtuIUNQpOgFFtlXJV41OltSsjW5jwAzxcko0MFkOIc7XuPjB" + - "XMrdjC9poYldrxNFrGOPFSyh19iS2UWKayKrtnhvDYAWrNCqOmRs01awEXBlwHp17VcLuze6XGCx7ZoPQX1Nu4uF1InAGpSm1B3pKtteeR" + - "WNNeLZjmNGNuiorHyxLTx1bQTfkG2UzZTTR0e2XatiXt5nCDxSqP2OkOxH7dew36fm9LpkFbmgtlxWxjHX8buYzSJCAjTqqwW3rHCEfQjv" + - "B4T7CTJrAgehCG9zL82P59DQbGXXWqRHbw5g9QszREQys1m56SHLosNptVPUwy7vD70rRf5s8knohW5npEZS9f3RGel64mj5g7bQBBkopx" + - "f6uac3MlJAe9d6C0B7fexZJABln2kCtXXYzITflICISwxuZ0YXHJmT2sMSIpn9VwMnMidV4JsM2BD8ykExZ5QyeVyOCXHDxvRvFwQwjQfR" + - "kkqQmtFREitKWl5njO8wLJw0XyeIVAej75NsGKKZWVjyaupaM9Bqn6NFrWjELFacLox6OCcRIDSDl3ntNN8tIzGOF7aXVCIqljJl0IL9Pz" + - "NenmmubaNm48YjfkBk8MqOUSYJYaFkO1qCKbVdMg7yTqKEHgSUqEkoFPoJMH6GAozC", - } - sc, _ := apiTest.PutConfig(*admin, cfg) - assert.Equal(200, sc, "the status code of modifying configurations with admin user should be 200") -} diff --git a/src/core/api/email.go b/src/core/api/email.go index 9f9909137..e20d37364 100644 --- a/src/core/api/email.go +++ b/src/core/api/email.go @@ -16,11 +16,12 @@ package api import ( "errors" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" "net" "strconv" "github.com/goharbor/harbor/src/common/utils/email" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" ) @@ -52,9 +53,10 @@ func (e *EmailAPI) Ping() { var host, username, password, identity string var port int var ssl, insecure bool + ctx := orm.Context() body := e.Ctx.Input.CopyBody(1 << 32) if body == nil || len(body) == 0 { - cfg, err := config.Email() + cfg, err := config.Email(ctx) if err != nil { log.Errorf("failed to get email configurations: %v", err) e.SendInternalServerError(err) @@ -88,7 +90,7 @@ func (e *EmailAPI) Ping() { } if settings.Password == nil { - cfg, err := config.Email() + cfg, err := config.Email(ctx) if err != nil { log.Errorf("failed to get email configurations: %v", err) e.SendInternalServerError(err) diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index 00761a0a8..a60d616d5 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -35,10 +35,11 @@ import ( "github.com/goharbor/harbor/src/common/job/test" "github.com/goharbor/harbor/src/common/models" testutils "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/controller/config" apimodels "github.com/goharbor/harbor/src/core/api/models" _ "github.com/goharbor/harbor/src/core/auth/db" _ "github.com/goharbor/harbor/src/core/auth/ldap" - "github.com/goharbor/harbor/src/core/config" + libOrm "github.com/goharbor/harbor/src/lib/orm" "github.com/goharbor/harbor/src/server/middleware" "github.com/goharbor/harbor/src/server/middleware/orm" "github.com/goharbor/harbor/src/server/middleware/security" @@ -77,12 +78,12 @@ type usrInfo struct { } func init() { - config.Init() testutils.InitDatabaseFromEnv() + config.Init() dao.PrepareTestData([]string{"delete from harbor_user where user_id >2", "delete from project where owner_id >2"}, []string{}) config.Upload(testutils.GetUnitTestConfig()) - allCfgs, _ := config.GetSystemCfg() + allCfgs, _ := config.GetSystemCfg(libOrm.Context()) testutils.TraceCfgMap(allCfgs) _, file, _, _ := runtime.Caller(0) @@ -106,8 +107,6 @@ func init() { beego.Router("/api/statistics", &StatisticAPI{}) beego.Router("/api/users/?:id", &UserAPI{}) beego.Router("/api/usergroups/?:ugid([0-9]+)", &UserGroupAPI{}) - beego.Router("/api/configurations", &ConfigAPI{}) - beego.Router("/api/configs", &ConfigAPI{}, "get:GetInternalConfig") beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping") beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List") beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete") @@ -677,82 +676,6 @@ func (a testapi) UsersDelete(userID int, authInfo usrInfo) (int, error) { return httpStatusCode, err } -// Post ldap test -func (a testapi) LdapPost(authInfo usrInfo, ldapConf apilib.LdapConf) (int, error) { - - _sling := sling.New().Post(a.basePath) - - // create path and map variables - path := "/api/ldap/ping" - - _sling = _sling.Path(path) - - // body params - _sling = _sling.BodyJSON(ldapConf) - httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo) - return httpStatusCode, err -} - -// Search Ldap Groups -func (a testapi) LdapGroupsSearch(groupName, groupDN string, authInfo ...usrInfo) (int, []apilib.LdapGroupsSearch, error) { - _sling := sling.New().Get(a.basePath) - // create path and map variables - path := "/api/ldap/groups/search" - _sling = _sling.Path(path) - // body params - type QueryParams struct { - GroupName string `url:"groupname, omitempty"` - GroupDN string `url:"groupdn, omitempty"` - } - _sling = _sling.QueryStruct(&QueryParams{GroupName: groupName, GroupDN: groupDN}) - httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo...) - var successPayLoad []apilib.LdapGroupsSearch - if 200 == httpStatusCode && nil == err { - err = json.Unmarshal(body, &successPayLoad) - } - return httpStatusCode, successPayLoad, err -} - -func (a testapi) GetConfig(authInfo usrInfo) (int, map[string]*value, error) { - _sling := sling.New().Base(a.basePath).Get("/api/configurations") - - cfg := map[string]*value{} - - code, body, err := request(_sling, jsonAcceptHeader, authInfo) - if err == nil && code == 200 { - err = json.Unmarshal(body, &cfg) - } - return code, cfg, err -} - -func (a testapi) GetInternalConfig(authInfo usrInfo) (int, map[string]interface{}, error) { - _sling := sling.New().Base(a.basePath).Get("/api/configs") - - cfg := map[string]interface{}{} - - code, body, err := request(_sling, jsonAcceptHeader, authInfo) - if err == nil && code == 200 { - err = json.Unmarshal(body, &cfg) - } - return code, cfg, err -} - -func (a testapi) PutConfig(authInfo usrInfo, cfg map[string]interface{}) (int, error) { - _sling := sling.New().Base(a.basePath).Put("/api/configurations").BodyJSON(cfg) - - code, _, err := request(_sling, jsonAcceptHeader, authInfo) - - return code, err -} - -func (a testapi) ResetConfig(authInfo usrInfo) (int, error) { - _sling := sling.New().Base(a.basePath).Post("/api/configurations/reset") - - code, _, err := request(_sling, jsonAcceptHeader, authInfo) - - return code, err -} - func (a testapi) PingEmail(authInfo usrInfo, settings []byte) (int, string, error) { _sling := sling.New().Base(a.basePath).Post("/api/email/ping").Body(bytes.NewReader(settings)) diff --git a/src/core/api/health.go b/src/core/api/health.go index adb12095a..694b5803a 100644 --- a/src/core/api/health.go +++ b/src/core/api/health.go @@ -17,6 +17,7 @@ package api import ( "errors" "fmt" + "github.com/goharbor/harbor/src/controller/config" "io/ioutil" "net/http" "sort" @@ -28,7 +29,6 @@ import ( "github.com/goharbor/harbor/src/common/dao" httputil "github.com/goharbor/harbor/src/common/http" "github.com/goharbor/harbor/src/common/utils" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/redis" ) diff --git a/src/core/api/internal.go b/src/core/api/internal.go index 4b0efe13b..53048e99b 100644 --- a/src/core/api/internal.go +++ b/src/core/api/internal.go @@ -16,13 +16,13 @@ package api import ( "context" + "github.com/goharbor/harbor/src/controller/config" o "github.com/astaxie/beego/orm" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/controller/quota" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/orm" @@ -78,46 +78,47 @@ func (ia *InternalAPI) SwitchQuota() { ia.SendBadRequestError(err) return } - cur := config.ReadOnly() + ctx := orm.NewContext(ia.Ctx.Request.Context(), o.NewOrm()) + cur := config.ReadOnly(ctx) // quota per project from disable to enable, it needs to update the quota usage bases on the DB records. - if !config.QuotaPerProjectEnable() && req.Enabled { + if !config.QuotaPerProjectEnable(ctx) && req.Enabled { if !cur { - config.GetCfgManager().Set(common.ReadOnly, true) - config.GetCfgManager().Save() + config.GetCfgManager(ctx).Set(ctx, common.ReadOnly, true) + config.GetCfgManager(ctx).Save(ctx) } - - ctx := orm.NewContext(ia.Ctx.Request.Context(), o.NewOrm()) if err := quota.RefreshForProjects(ctx); err != nil { ia.SendInternalServerError(err) return } } defer func() { - config.GetCfgManager().Set(common.ReadOnly, cur) - config.GetCfgManager().Set(common.QuotaPerProjectEnable, req.Enabled) - config.GetCfgManager().Save() + ctx := orm.Context() + config.GetCfgManager(ctx).Set(ctx, common.ReadOnly, cur) + config.GetCfgManager(ctx).Set(ctx, common.QuotaPerProjectEnable, req.Enabled) + config.GetCfgManager(ctx).Save(ctx) }() return } // SyncQuota ... func (ia *InternalAPI) SyncQuota() { - if !config.QuotaPerProjectEnable() { + if !config.QuotaPerProjectEnable(orm.Context()) { ia.SendError(errors.ForbiddenError(nil).WithMessage("quota per project is disabled")) return } - - cur := config.ReadOnly() - cfgMgr := config.GetCfgManager() + ctx := orm.Context() + cur := config.ReadOnly(ctx) + cfgMgr := config.GetCfgManager(ctx) if !cur { - cfgMgr.Set(common.ReadOnly, true) - cfgMgr.Save() + cfgMgr.Set(ctx, common.ReadOnly, true) + cfgMgr.Save(ctx) } // For api call, to avoid the timeout, it should be asynchronous go func() { defer func() { - cfgMgr.Set(common.ReadOnly, cur) - cfgMgr.Save() + ctx := orm.Context() + cfgMgr.Set(ctx, common.ReadOnly, cur) + cfgMgr.Save(ctx) }() log.Info("start to sync quota(API), the system will be set to ReadOnly and back it normal once it done.") ctx := orm.NewContext(context.TODO(), o.NewOrm()) diff --git a/src/core/api/projectmember.go b/src/core/api/projectmember.go index bba85bfc5..f5465bcf9 100644 --- a/src/core/api/projectmember.go +++ b/src/core/api/projectmember.go @@ -16,6 +16,8 @@ package api import ( "fmt" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" "net/http" "strconv" "strings" @@ -27,7 +29,6 @@ import ( "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/core/auth" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/errors" ) @@ -98,7 +99,7 @@ func (pma *ProjectMemberAPI) Prepare() { pma.member = members[0] } - authMode, err := config.AuthMode() + authMode, err := config.AuthMode(orm.Context()) if err != nil { pma.SendInternalServerError(fmt.Errorf("failed to get authentication mode")) } diff --git a/src/core/api/user.go b/src/core/api/user.go index cba344163..2035b47bc 100644 --- a/src/core/api/user.go +++ b/src/core/api/user.go @@ -19,6 +19,8 @@ import ( "errors" "fmt" "github.com/goharbor/harbor/src/common/rbac/system" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" "net/http" "regexp" "strconv" @@ -29,7 +31,6 @@ import ( "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/security/local" "github.com/goharbor/harbor/src/common/utils" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/permission/types" @@ -63,7 +64,7 @@ type secretReq struct { // Prepare validates the URL and parms func (ua *UserAPI) Prepare() { ua.BaseController.Prepare() - mode, err := config.AuthMode() + mode, err := config.AuthMode(orm.Context()) if err != nil { log.Errorf("failed to get auth mode: %v", err) ua.SendInternalServerError(errors.New("")) @@ -81,7 +82,7 @@ func (ua *UserAPI) Prepare() { ua.secretKey = key } - self, err := config.SelfRegistration() + self, err := config.SelfRegistration(orm.Context()) if err != nil { log.Errorf("failed to get self registration: %v", err) ua.SendInternalServerError(errors.New("")) diff --git a/src/core/api/user_test.go b/src/core/api/user_test.go index e142fd45e..930e788ea 100644 --- a/src/core/api/user_test.go +++ b/src/core/api/user_test.go @@ -16,6 +16,7 @@ package api import ( "context" "fmt" + "github.com/goharbor/harbor/src/controller/config" securitytesting "github.com/goharbor/harbor/src/testing/common/security" "github.com/goharbor/harbor/src/testing/mock" "net/http" @@ -31,7 +32,6 @@ import ( "github.com/astaxie/beego" "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/core/config" ) var testUser0002ID, testUser0003ID int diff --git a/src/core/api/usergroup.go b/src/core/api/usergroup.go index 01b581486..f99468615 100644 --- a/src/core/api/usergroup.go +++ b/src/core/api/usergroup.go @@ -19,6 +19,8 @@ import ( "fmt" "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/rbac/system" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" "net/http" "strconv" "strings" @@ -27,7 +29,6 @@ import ( "github.com/goharbor/harbor/src/common/dao/group" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/core/auth" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/ldap" ) @@ -60,7 +61,7 @@ func (uga *UserGroupAPI) Prepare() { return } uga.id = int(ugid) - authMode, err := config.AuthMode() + authMode, err := config.AuthMode(orm.Context()) if err != nil { uga.SendInternalServerError(errors.New("failed to get authentication mode")) } diff --git a/src/core/auth/authenticator.go b/src/core/auth/authenticator.go index 5c83838cf..44e66b485 100644 --- a/src/core/auth/authenticator.go +++ b/src/core/auth/authenticator.go @@ -17,12 +17,13 @@ package auth import ( "errors" "fmt" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" "time" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" ) @@ -129,7 +130,7 @@ func Register(name string, h AuthenticateHelper) { // Login authenticates user credentials based on setting. func Login(m models.AuthModel) (*models.User, error) { - authMode, err := config.AuthMode() + authMode, err := config.AuthMode(orm.Context()) if err != nil { return nil, err } @@ -160,7 +161,7 @@ func Login(m models.AuthModel) (*models.User, error) { } func getHelper() (AuthenticateHelper, error) { - authMode, err := config.AuthMode() + authMode, err := config.AuthMode(orm.Context()) if err != nil { return nil, err } diff --git a/src/core/auth/authproxy/auth.go b/src/core/auth/authproxy/auth.go index ddd667f82..c82bcf407 100644 --- a/src/core/auth/authproxy/auth.go +++ b/src/core/auth/authproxy/auth.go @@ -20,7 +20,10 @@ import ( "encoding/json" "errors" "fmt" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/jobservice/logger" + cfgModels "github.com/goharbor/harbor/src/lib/config/models" + "github.com/goharbor/harbor/src/lib/orm" "io/ioutil" "net/http" "strings" @@ -33,7 +36,6 @@ import ( "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/core/auth" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/authproxy" ) @@ -108,7 +110,7 @@ func (a *Auth) Authenticate(m models.AuthModel) (*models.User, error) { } func (a *Auth) tokenReview(sessionID string) (*models.User, error) { - httpAuthProxySetting, err := config.HTTPAuthProxySetting() + httpAuthProxySetting, err := config.HTTPAuthProxySetting(orm.Context()) if err != nil { return nil, err } @@ -216,7 +218,7 @@ func (a *Auth) ensure() error { a.client = &http.Client{} } if time.Now().Sub(a.settingTimeStamp) >= refreshDuration { - setting, err := config.HTTPAuthProxySetting() + setting, err := config.HTTPAuthProxySetting(orm.Context()) if err != nil { return err } @@ -233,7 +235,7 @@ func (a *Auth) ensure() error { return nil } -func getTLSConfig(setting *models.HTTPAuthProxy) (*tls.Config, error) { +func getTLSConfig(setting *cfgModels.HTTPAuthProxy) (*tls.Config, error) { c := setting.ServerCertificate if setting.VerifyCert && len(c) > 0 { certs := x509.NewCertPool() diff --git a/src/core/auth/authproxy/auth_test.go b/src/core/auth/authproxy/auth_test.go index 498d96142..f58ba3307 100644 --- a/src/core/auth/authproxy/auth_test.go +++ b/src/core/auth/authproxy/auth_test.go @@ -20,9 +20,10 @@ import ( "github.com/goharbor/harbor/src/common/dao/group" "github.com/goharbor/harbor/src/common/models" cut "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/core/auth" "github.com/goharbor/harbor/src/core/auth/authproxy/test" - "github.com/goharbor/harbor/src/core/config" + cfgModels "github.com/goharbor/harbor/src/lib/config/models" "github.com/stretchr/testify/assert" "net/http/httptest" "os" @@ -214,11 +215,11 @@ func TestGetTLSConfig(t *testing.T) { nilRootCA bool } cases := []struct { - input *models.HTTPAuthProxy + input *cfgModels.HTTPAuthProxy expect result }{ { - input: &models.HTTPAuthProxy{ + input: &cfgModels.HTTPAuthProxy{ Endpoint: "https://127.0.0.1/login", TokenReviewEndpoint: "https://127.0.0.1/tokenreview", VerifyCert: false, @@ -232,7 +233,7 @@ func TestGetTLSConfig(t *testing.T) { }, }, { - input: &models.HTTPAuthProxy{ + input: &cfgModels.HTTPAuthProxy{ Endpoint: "https://127.0.0.1/login", TokenReviewEndpoint: "https://127.0.0.1/tokenreview", VerifyCert: false, @@ -246,7 +247,7 @@ func TestGetTLSConfig(t *testing.T) { }, }, { - input: &models.HTTPAuthProxy{ + input: &cfgModels.HTTPAuthProxy{ Endpoint: "https://127.0.0.1/login", TokenReviewEndpoint: "https://127.0.0.1/tokenreview", VerifyCert: true, @@ -258,7 +259,7 @@ func TestGetTLSConfig(t *testing.T) { }, }, { - input: &models.HTTPAuthProxy{ + input: &cfgModels.HTTPAuthProxy{ Endpoint: "https://127.0.0.1/login", TokenReviewEndpoint: "https://127.0.0.1/tokenreview", VerifyCert: true, @@ -272,7 +273,7 @@ func TestGetTLSConfig(t *testing.T) { }, }, { - input: &models.HTTPAuthProxy{ + input: &cfgModels.HTTPAuthProxy{ Endpoint: "https://127.0.0.1/login", TokenReviewEndpoint: "https://127.0.0.1/tokenreview", VerifyCert: true, diff --git a/src/core/auth/db/db_test.go b/src/core/auth/db/db_test.go index cdea76c4f..3ec8a76e2 100644 --- a/src/core/auth/db/db_test.go +++ b/src/core/auth/db/db_test.go @@ -14,6 +14,7 @@ package db import ( + "github.com/goharbor/harbor/src/controller/config" "log" "os" "testing" @@ -24,8 +25,6 @@ import ( "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/core/auth" - "github.com/goharbor/harbor/src/core/config" - coreConfig "github.com/goharbor/harbor/src/core/config" ) var testConfig = map[string]interface{}{ @@ -62,7 +61,7 @@ func TestMain(m *testing.M) { log.Fatalf("failed to set env %s: %v", "KEY_PATH", err) } - coreConfig.Init() + config.Init() config.Upload(testConfig) retCode := m.Run() diff --git a/src/core/auth/ldap/ldap.go b/src/core/auth/ldap/ldap.go index 864e4c377..09f77ef30 100644 --- a/src/core/auth/ldap/ldap.go +++ b/src/core/auth/ldap/ldap.go @@ -17,6 +17,8 @@ package ldap import ( "context" "fmt" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" "github.com/goharbor/harbor/src/pkg/ldap/model" "regexp" "strings" @@ -32,7 +34,6 @@ import ( "github.com/goharbor/harbor/src/pkg/ldap" "github.com/goharbor/harbor/src/core/auth" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" ) @@ -45,13 +46,13 @@ type Auth struct { // if the check is successful a dummy record will be inserted into DB, such that this user can // be associated to other entities in the system. func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { - + ctx := orm.Context() p := m.Principal if len(strings.TrimSpace(p)) == 0 { log.Debugf("LDAP authentication failed for empty user id.") return nil, auth.NewErrAuth("Empty user id") } - ldapSession, err := ldapCtl.Ctl.Session(context.Background()) + ldapSession, err := ldapCtl.Ctl.Session(ctx) if err != nil { return nil, fmt.Errorf("can not load system ldap config: %v", err) } @@ -88,15 +89,15 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { u.Email = strings.TrimSpace(ldapUsers[0].Email) l.syncUserInfoFromDB(&u) - l.attachLDAPGroup(ldapUsers, &u, ldapSession) + l.attachLDAPGroup(ctx, ldapUsers, &u, ldapSession) return &u, nil } -func (l *Auth) attachLDAPGroup(ldapUsers []model.User, u *models.User, sess *ldap.Session) { +func (l *Auth) attachLDAPGroup(ctx context.Context, ldapUsers []model.User, u *models.User, sess *ldap.Session) { // Retrieve ldap related info in login to avoid too many traffic with LDAP server. // Get group admin dn - groupCfg, err := config.LDAPGroupConf() + groupCfg, err := config.LDAPGroupConf(ctx) if err != nil { log.Warningf("Failed to fetch ldap group configuration:%v", err) // most likely user doesn't configure user group info, it should not block user login @@ -161,7 +162,7 @@ func (l *Auth) OnBoardUser(u *models.User) error { // SearchUser -- Search user in ldap func (l *Auth) SearchUser(username string) (*models.User, error) { var user models.User - s, err := ldapCtl.Ctl.Session(context.Background()) + s, err := ldapCtl.Ctl.Session(orm.Context()) if err != nil { return nil, err } @@ -196,7 +197,7 @@ func (l *Auth) SearchGroup(groupKey string) (*models.UserGroup, error) { if _, err := goldap.ParseDN(groupKey); err != nil { return nil, auth.ErrInvalidLDAPGroupDN } - s, err := ldapCtl.Ctl.Session(context.Background()) + s, err := ldapCtl.Ctl.Session(orm.Context()) if err != nil { return nil, fmt.Errorf("can not load system ldap config: %v", err) diff --git a/src/core/auth/ldap/ldap_test.go b/src/core/auth/ldap/ldap_test.go index 5cd1c799b..435ce4cd6 100644 --- a/src/core/auth/ldap/ldap_test.go +++ b/src/core/auth/ldap/ldap_test.go @@ -14,6 +14,7 @@ package ldap import ( + "github.com/goharbor/harbor/src/controller/config" "github.com/stretchr/testify/assert" // "fmt" // "strings" @@ -30,7 +31,6 @@ import ( "github.com/goharbor/harbor/src/common/dao/group" "github.com/goharbor/harbor/src/core/auth" - coreConfig "github.com/goharbor/harbor/src/core/config" ) var ldapTestConfig = map[string]interface{}{ @@ -61,7 +61,7 @@ var ldapTestConfig = map[string]interface{}{ func TestMain(m *testing.M) { test.InitDatabaseFromEnv() - coreConfig.InitWithSettings(ldapTestConfig) + config.InitWithSettings(ldapTestConfig) secretKeyPath := "/tmp/secretkey" _, err := test.GenerateKey(secretKeyPath) diff --git a/src/core/auth/uaa/uaa.go b/src/core/auth/uaa/uaa.go index ebc832fb2..06e496017 100644 --- a/src/core/auth/uaa/uaa.go +++ b/src/core/auth/uaa/uaa.go @@ -16,6 +16,8 @@ package uaa import ( "fmt" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" "os" "strings" "sync" @@ -25,7 +27,6 @@ import ( "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/uaa" "github.com/goharbor/harbor/src/core/auth" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" ) @@ -129,7 +130,7 @@ func (u *Auth) SearchUser(username string) (*models.User, error) { func (u *Auth) ensureClient() error { var cfg *uaa.ClientConfig - UAASettings, err := config.UAASettings() + UAASettings, err := config.UAASettings(orm.Context()) // log.Debugf("Uaa settings: %+v", UAASettings) if err != nil { log.Warningf("Failed to get UAA setting from Admin Server, error: %v", err) diff --git a/src/core/auth/uaa/uaa_test.go b/src/core/auth/uaa/uaa_test.go index 6219958a5..4684977ec 100644 --- a/src/core/auth/uaa/uaa_test.go +++ b/src/core/auth/uaa/uaa_test.go @@ -15,6 +15,7 @@ package uaa import ( + "github.com/goharbor/harbor/src/controller/config" "os" "testing" @@ -22,7 +23,6 @@ import ( "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/test" "github.com/goharbor/harbor/src/common/utils/uaa" - "github.com/goharbor/harbor/src/core/config" "github.com/stretchr/testify/assert" ) diff --git a/src/core/config/config.go b/src/core/config/config.go deleted file mode 100755 index 122196d6b..000000000 --- a/src/core/config/config.go +++ /dev/null @@ -1,491 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package config provide config for core api and other modules -// Before accessing user settings, need to call Load() -// For system settings, no need to call Load() -package config - -import ( - "errors" - "os" - "strconv" - "strings" - - "github.com/goharbor/harbor/src/common" - comcfg "github.com/goharbor/harbor/src/common/config" - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/secret" - "github.com/goharbor/harbor/src/lib/log" - "github.com/goharbor/harbor/src/pkg/ldap/model" -) - -const ( - defaultKeyPath = "/etc/core/key" - defaultRegistryTokenPrivateKeyPath = "/etc/core/private_key.pem" - - // SessionCookieName is the name of the cookie for session ID - SessionCookieName = "sid" -) - -var ( - // SecretStore manages secrets - SecretStore *secret.Store - keyProvider comcfg.KeyProvider - // defined as a var for testing. - defaultCACertPath = "/etc/core/ca/ca.crt" - cfgMgr *comcfg.CfgManager -) - -// Init configurations -func Init() { - // init key provider - initKeyProvider() - - cfgMgr = comcfg.NewDBCfgManager() - - log.Info("init secret store") - // init secret store - initSecretStore() -} - -// InitWithSettings init config with predefined configs, and optionally overwrite the keyprovider -func InitWithSettings(cfgs map[string]interface{}, kp ...comcfg.KeyProvider) { - Init() - cfgMgr = comcfg.NewInMemoryManager() - cfgMgr.UpdateConfig(cfgs) - if len(kp) > 0 { - keyProvider = kp[0] - } -} - -func initKeyProvider() { - path := os.Getenv("KEY_PATH") - if len(path) == 0 { - path = defaultKeyPath - } - log.Infof("key path: %s", path) - - keyProvider = comcfg.NewFileKeyProvider(path) -} - -func initSecretStore() { - m := map[string]string{} - m[JobserviceSecret()] = secret.JobserviceUser - SecretStore = secret.NewStore(m) -} - -// GetCfgManager return the current config manager -func GetCfgManager() *comcfg.CfgManager { - if cfgMgr == nil { - return comcfg.NewDBCfgManager() - } - return cfgMgr -} - -// Load configurations -func Load() error { - return cfgMgr.Load() -} - -// Upload save all system configurations -func Upload(cfg map[string]interface{}) error { - return cfgMgr.UpdateConfig(cfg) -} - -// GetSystemCfg returns the system configurations -func GetSystemCfg() (map[string]interface{}, error) { - sysCfg := cfgMgr.GetAll() - if len(sysCfg) == 0 { - return nil, errors.New("can not load system config, the database might be down") - } - return sysCfg, nil -} - -// AuthMode ... -func AuthMode() (string, error) { - err := cfgMgr.Load() - if err != nil { - log.Errorf("failed to load config, error %v", err) - return "db_auth", err - } - return cfgMgr.Get(common.AUTHMode).GetString(), nil -} - -// TokenPrivateKeyPath returns the path to the key for signing token for registry -func TokenPrivateKeyPath() string { - path := os.Getenv("TOKEN_PRIVATE_KEY_PATH") - if len(path) == 0 { - path = defaultRegistryTokenPrivateKeyPath - } - return path -} - -// LDAPConf returns the setting of ldap server -func LDAPConf() (*model.LdapConf, error) { - err := cfgMgr.Load() - if err != nil { - return nil, err - } - return &model.LdapConf{ - URL: cfgMgr.Get(common.LDAPURL).GetString(), - SearchDn: cfgMgr.Get(common.LDAPSearchDN).GetString(), - SearchPassword: cfgMgr.Get(common.LDAPSearchPwd).GetString(), - BaseDn: cfgMgr.Get(common.LDAPBaseDN).GetString(), - UID: cfgMgr.Get(common.LDAPUID).GetString(), - Filter: cfgMgr.Get(common.LDAPFilter).GetString(), - Scope: cfgMgr.Get(common.LDAPScope).GetInt(), - ConnectionTimeout: cfgMgr.Get(common.LDAPTimeout).GetInt(), - VerifyCert: cfgMgr.Get(common.LDAPVerifyCert).GetBool(), - }, nil -} - -// LDAPGroupConf returns the setting of ldap group search -func LDAPGroupConf() (*model.GroupConf, error) { - err := cfgMgr.Load() - if err != nil { - return nil, err - } - return &model.GroupConf{ - BaseDN: cfgMgr.Get(common.LDAPGroupBaseDN).GetString(), - Filter: cfgMgr.Get(common.LDAPGroupSearchFilter).GetString(), - NameAttribute: cfgMgr.Get(common.LDAPGroupAttributeName).GetString(), - SearchScope: cfgMgr.Get(common.LDAPGroupSearchScope).GetInt(), - AdminDN: cfgMgr.Get(common.LDAPGroupAdminDn).GetString(), - MembershipAttribute: cfgMgr.Get(common.LDAPGroupMembershipAttribute).GetString(), - }, nil -} - -// TokenExpiration returns the token expiration time (in minute) -func TokenExpiration() (int, error) { - return cfgMgr.Get(common.TokenExpiration).GetInt(), nil -} - -// RobotTokenDuration returns the token expiration time of robot account (in minute) -func RobotTokenDuration() int { - return cfgMgr.Get(common.RobotTokenDuration).GetInt() -} - -// ExtEndpoint returns the external URL of Harbor: protocol://host:port -func ExtEndpoint() (string, error) { - return cfgMgr.Get(common.ExtEndpoint).GetString(), nil -} - -// ExtURL returns the external URL: host:port -func ExtURL() (string, error) { - endpoint, err := ExtEndpoint() - if err != nil { - log.Errorf("failed to load config, error %v", err) - } - l := strings.Split(endpoint, "://") - if len(l) > 1 { - return l[1], nil - } - return endpoint, nil -} - -// SecretKey returns the secret key to encrypt the password of target -func SecretKey() (string, error) { - return keyProvider.Get(nil) -} - -// SelfRegistration returns the enablement of self registration -func SelfRegistration() (bool, error) { - return cfgMgr.Get(common.SelfRegistration).GetBool(), nil -} - -// RegistryURL ... -func RegistryURL() (string, error) { - url := os.Getenv("REGISTRY_URL") - if len(url) == 0 { - url = "http://registry:5000" - } - return url, nil -} - -// InternalJobServiceURL returns jobservice URL for internal communication between Harbor containers -func InternalJobServiceURL() string { - return os.Getenv("JOBSERVICE_URL") -} - -// GetCoreURL returns the url of core from env -func GetCoreURL() string { - return os.Getenv("CORE_URL") -} - -// InternalCoreURL returns the local harbor core url -func InternalCoreURL() string { - return strings.TrimSuffix(cfgMgr.Get(common.CoreURL).GetString(), "/") -} - -// LocalCoreURL returns the local harbor core url -func LocalCoreURL() string { - return cfgMgr.Get(common.CoreLocalURL).GetString() -} - -// InternalTokenServiceEndpoint returns token service endpoint for internal communication between Harbor containers -func InternalTokenServiceEndpoint() string { - return InternalCoreURL() + "/service/token" -} - -// InternalNotaryEndpoint returns notary server endpoint for internal communication between Harbor containers -// This is currently a conventional value and can be unaccessible when Harbor is not deployed with Notary. -func InternalNotaryEndpoint() string { - return cfgMgr.Get(common.NotaryURL).GetString() -} - -// InitialAdminPassword returns the initial password for administrator -func InitialAdminPassword() (string, error) { - return cfgMgr.Get(common.AdminInitialPassword).GetString(), nil -} - -// OnlyAdminCreateProject returns the flag to restrict that only sys admin can create project -func OnlyAdminCreateProject() (bool, error) { - return cfgMgr.Get(common.ProjectCreationRestriction).GetString() == common.ProCrtRestrAdmOnly, nil -} - -// Email returns email server settings -func Email() (*models.Email, error) { - err := cfgMgr.Load() - if err != nil { - return nil, err - } - return &models.Email{ - Host: cfgMgr.Get(common.EmailHost).GetString(), - Port: cfgMgr.Get(common.EmailPort).GetInt(), - Username: cfgMgr.Get(common.EmailUsername).GetString(), - Password: cfgMgr.Get(common.EmailPassword).GetString(), - SSL: cfgMgr.Get(common.EmailSSL).GetBool(), - From: cfgMgr.Get(common.EmailFrom).GetString(), - Identity: cfgMgr.Get(common.EmailIdentity).GetString(), - Insecure: cfgMgr.Get(common.EmailInsecure).GetBool(), - }, nil -} - -// Database returns database settings -func Database() (*models.Database, error) { - database := &models.Database{} - database.Type = cfgMgr.Get(common.DatabaseType).GetString() - postgresql := &models.PostGreSQL{ - Host: cfgMgr.Get(common.PostGreSQLHOST).GetString(), - Port: cfgMgr.Get(common.PostGreSQLPort).GetInt(), - Username: cfgMgr.Get(common.PostGreSQLUsername).GetString(), - Password: cfgMgr.Get(common.PostGreSQLPassword).GetString(), - Database: cfgMgr.Get(common.PostGreSQLDatabase).GetString(), - SSLMode: cfgMgr.Get(common.PostGreSQLSSLMode).GetString(), - MaxIdleConns: cfgMgr.Get(common.PostGreSQLMaxIdleConns).GetInt(), - MaxOpenConns: cfgMgr.Get(common.PostGreSQLMaxOpenConns).GetInt(), - } - database.PostGreSQL = postgresql - - return database, nil -} - -// CoreSecret returns a secret to mark harbor-core when communicate with -// other component -func CoreSecret() string { - return os.Getenv("CORE_SECRET") -} - -// RegistryCredential returns the username and password the core uses to access registry -func RegistryCredential() (string, string) { - return os.Getenv("REGISTRY_CREDENTIAL_USERNAME"), os.Getenv("REGISTRY_CREDENTIAL_PASSWORD") -} - -// JobserviceSecret returns a secret to mark Jobservice when communicate with -// other component -// TODO replace it with method of SecretStore -func JobserviceSecret() string { - return os.Getenv("JOBSERVICE_SECRET") -} - -// WithNotary returns a bool value to indicate if Harbor's deployed with Notary -func WithNotary() bool { - return cfgMgr.Get(common.WithNotary).GetBool() -} - -// WithTrivy returns a bool value to indicate if Harbor's deployed with Trivy. -func WithTrivy() bool { - return cfgMgr.Get(common.WithTrivy).GetBool() -} - -// TrivyAdapterURL returns the endpoint URL of a Trivy adapter instance, by default it's the one deployed within Harbor. -func TrivyAdapterURL() string { - return cfgMgr.Get(common.TrivyAdapterURL).GetString() -} - -// UAASettings returns the UAASettings to access UAA service. -func UAASettings() (*models.UAASettings, error) { - err := cfgMgr.Load() - if err != nil { - return nil, err - } - us := &models.UAASettings{ - Endpoint: cfgMgr.Get(common.UAAEndpoint).GetString(), - ClientID: cfgMgr.Get(common.UAAClientID).GetString(), - ClientSecret: cfgMgr.Get(common.UAAClientSecret).GetString(), - VerifyCert: cfgMgr.Get(common.UAAVerifyCert).GetBool(), - } - return us, nil -} - -// ReadOnly returns a bool to indicates if Harbor is in read only mode. -func ReadOnly() bool { - return cfgMgr.Get(common.ReadOnly).GetBool() -} - -// WithChartMuseum returns a bool to indicate if chartmuseum is deployed with Harbor. -func WithChartMuseum() bool { - return cfgMgr.Get(common.WithChartMuseum).GetBool() -} - -// GetChartMuseumEndpoint returns the endpoint of the chartmuseum service -// otherwise an non nil error is returned -func GetChartMuseumEndpoint() (string, error) { - chartEndpoint := strings.TrimSpace(cfgMgr.Get(common.ChartRepoURL).GetString()) - if len(chartEndpoint) == 0 { - return "", errors.New("empty chartmuseum endpoint") - } - return chartEndpoint, nil -} - -// GetRedisOfRegURL returns the URL of Redis used by registry -func GetRedisOfRegURL() string { - return os.Getenv("_REDIS_URL_REG") -} - -// GetPortalURL returns the URL of portal -func GetPortalURL() string { - url := os.Getenv("PORTAL_URL") - if len(url) == 0 { - return common.DefaultPortalURL - } - return url -} - -// GetRegistryCtlURL returns the URL of registryctl -func GetRegistryCtlURL() string { - url := os.Getenv("REGISTRY_CONTROLLER_URL") - if len(url) == 0 { - return common.DefaultRegistryCtlURL - } - return url -} - -// HTTPAuthProxySetting returns the setting of HTTP Auth proxy. the settings are only meaningful when the auth_mode is -// set to http_auth -func HTTPAuthProxySetting() (*models.HTTPAuthProxy, error) { - if err := cfgMgr.Load(); err != nil { - return nil, err - } - return &models.HTTPAuthProxy{ - Endpoint: cfgMgr.Get(common.HTTPAuthProxyEndpoint).GetString(), - TokenReviewEndpoint: cfgMgr.Get(common.HTTPAuthProxyTokenReviewEndpoint).GetString(), - AdminGroups: splitAndTrim(cfgMgr.Get(common.HTTPAuthProxyAdminGroups).GetString(), ","), - AdminUsernames: splitAndTrim(cfgMgr.Get(common.HTTPAuthProxyAdminUsernames).GetString(), ","), - VerifyCert: cfgMgr.Get(common.HTTPAuthProxyVerifyCert).GetBool(), - SkipSearch: cfgMgr.Get(common.HTTPAuthProxySkipSearch).GetBool(), - ServerCertificate: cfgMgr.Get(common.HTTPAuthProxyServerCertificate).GetString(), - }, nil -} - -// OIDCSetting returns the setting of OIDC provider, currently there's only one OIDC provider allowed for Harbor and it's -// only effective when auth_mode is set to oidc_auth -func OIDCSetting() (*models.OIDCSetting, error) { - if err := cfgMgr.Load(); err != nil { - return nil, err - } - scopeStr := cfgMgr.Get(common.OIDCScope).GetString() - extEndpoint := strings.TrimSuffix(cfgMgr.Get(common.ExtEndpoint).GetString(), "/") - scope := splitAndTrim(scopeStr, ",") - return &models.OIDCSetting{ - Name: cfgMgr.Get(common.OIDCName).GetString(), - Endpoint: cfgMgr.Get(common.OIDCEndpoint).GetString(), - VerifyCert: cfgMgr.Get(common.OIDCVerifyCert).GetBool(), - AutoOnboard: cfgMgr.Get(common.OIDCAutoOnboard).GetBool(), - ClientID: cfgMgr.Get(common.OIDCCLientID).GetString(), - ClientSecret: cfgMgr.Get(common.OIDCClientSecret).GetString(), - GroupsClaim: cfgMgr.Get(common.OIDCGroupsClaim).GetString(), - AdminGroup: cfgMgr.Get(common.OIDCAdminGroup).GetString(), - RedirectURL: extEndpoint + common.OIDCCallbackPath, - Scope: scope, - UserClaim: cfgMgr.Get(common.OIDCUserClaim).GetString(), - ExtraRedirectParms: cfgMgr.Get(common.OIDCExtraRedirectParms).GetStringToStringMap(), - }, nil -} - -// NotificationEnable returns a bool to indicates if notification enabled in harbor -func NotificationEnable() bool { - return cfgMgr.Get(common.NotificationEnable).GetBool() -} - -// QuotaPerProjectEnable returns a bool to indicates if quota per project enabled in harbor -func QuotaPerProjectEnable() bool { - return cfgMgr.Get(common.QuotaPerProjectEnable).GetBool() -} - -// QuotaSetting returns the setting of quota. -func QuotaSetting() (*models.QuotaSetting, error) { - if err := cfgMgr.Load(); err != nil { - return nil, err - } - return &models.QuotaSetting{ - StoragePerProject: cfgMgr.Get(common.StoragePerProject).GetInt64(), - }, nil -} - -// GetPermittedRegistryTypesForProxyCache returns the permitted registry types for proxy cache -func GetPermittedRegistryTypesForProxyCache() []string { - types := os.Getenv("PERMITTED_REGISTRY_TYPES_FOR_PROXY_CACHE") - if len(types) == 0 { - return []string{} - } - return strings.Split(types, ",") -} - -// GetGCTimeWindow returns the reserve time window of blob. -func GetGCTimeWindow() int64 { - // the env is for testing/debugging. For production, Do NOT set it. - if env, exist := os.LookupEnv("GC_TIME_WINDOW_HOURS"); exist { - timeWindow, err := strconv.ParseInt(env, 10, 64) - if err == nil { - return timeWindow - } - } - return common.DefaultGCTimeWindowHours -} - -// RobotPrefix user defined robot name prefix. -func RobotPrefix() string { - return cfgMgr.Get(common.RobotNamePrefix).GetString() -} - -// Metric returns the overall metric settings -func Metric() *models.Metric { - return &models.Metric{ - Enabled: cfgMgr.Get(common.MetricEnable).GetBool(), - Port: cfgMgr.Get(common.MetricPort).GetInt(), - Path: cfgMgr.Get(common.MetricPath).GetString(), - } -} - -func splitAndTrim(s, sep string) []string { - res := make([]string, 0) - for _, s := range strings.Split(s, sep) { - if e := strings.TrimSpace(s); len(e) > 0 { - res = append(res, e) - } - } - return res -} diff --git a/src/core/controllers/authproxy_redirect.go b/src/core/controllers/authproxy_redirect.go index e88295c95..995daec78 100644 --- a/src/core/controllers/authproxy_redirect.go +++ b/src/core/controllers/authproxy_redirect.go @@ -2,12 +2,13 @@ package controllers import ( "fmt" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" "net/http" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/core/api" "github.com/goharbor/harbor/src/core/auth/authproxy" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" ) @@ -25,7 +26,7 @@ type AuthProxyController struct { // Prepare checks the auth mode and fail early func (apc *AuthProxyController) Prepare() { - am, err := config.AuthMode() + am, err := config.AuthMode(orm.Context()) if err != nil { apc.SendInternalServerError(err) return diff --git a/src/core/controllers/base.go b/src/core/controllers/base.go index 2dbef147a..4d8155918 100644 --- a/src/core/controllers/base.go +++ b/src/core/controllers/base.go @@ -17,6 +17,8 @@ package controllers import ( "bytes" "context" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" "html/template" "net" "net/http" @@ -35,7 +37,6 @@ import ( email_util "github.com/goharbor/harbor/src/common/utils/email" "github.com/goharbor/harbor/src/core/api" "github.com/goharbor/harbor/src/core/auth" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/log" ) @@ -123,7 +124,7 @@ func (cc *CommonController) LogOut() { // UserExists checks if user exists when user input value in sign in form. func (cc *CommonController) UserExists() { - flag, err := config.SelfRegistration() + flag, err := config.SelfRegistration(orm.Context()) if err != nil { log.Errorf("Failed to get the status of self registration flag, error: %v, disabling user existence check", err) } @@ -216,7 +217,7 @@ func (cc *CommonController) SendResetEmail() { cc.CustomAbort(http.StatusInternalServerError, "internal_error") } - settings, err := config.Email() + settings, err := config.Email(orm.Context()) if err != nil { log.Errorf("failed to get email configurations: %v", err) cc.CustomAbort(http.StatusInternalServerError, "internal_error") @@ -281,7 +282,7 @@ func isUserResetable(u *models.User) bool { if u == nil { return false } - mode, err := config.AuthMode() + mode, err := config.AuthMode(orm.Context()) if err != nil { log.Errorf("Failed to get the auth mode, error: %v", err) return false diff --git a/src/core/controllers/controllers_test.go b/src/core/controllers/controllers_test.go index 2d9bf7b0f..e1640a2b0 100644 --- a/src/core/controllers/controllers_test.go +++ b/src/core/controllers/controllers_test.go @@ -14,10 +14,11 @@ package controllers import ( - "context" "fmt" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/core/middlewares" "github.com/goharbor/harbor/src/lib" + "github.com/goharbor/harbor/src/lib/orm" "net/http" "net/http/httptest" "os" @@ -30,7 +31,6 @@ import ( "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/models" utilstest "github.com/goharbor/harbor/src/common/utils/test" - "github.com/goharbor/harbor/src/core/config" "github.com/stretchr/testify/assert" ) @@ -86,9 +86,9 @@ func TestUserResettable(t *testing.T) { } func TestRedirectForOIDC(t *testing.T) { - ctx := lib.WithAuthMode(context.Background(), common.DBAuth) + ctx := lib.WithAuthMode(orm.Context(), common.DBAuth) assert.False(t, redirectForOIDC(ctx, "nonexist")) - ctx = lib.WithAuthMode(context.Background(), common.OIDCAuth) + ctx = lib.WithAuthMode(orm.Context(), common.OIDCAuth) assert.True(t, redirectForOIDC(ctx, "nonexist")) assert.False(t, redirectForOIDC(ctx, "admin")) diff --git a/src/core/controllers/oidc.go b/src/core/controllers/oidc.go index 7fff8f3b9..958e2d28a 100644 --- a/src/core/controllers/oidc.go +++ b/src/core/controllers/oidc.go @@ -17,6 +17,8 @@ package controllers import ( "encoding/json" "fmt" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" "net/http" "strings" @@ -25,7 +27,6 @@ import ( "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/core/api" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/oidc" @@ -47,7 +48,7 @@ type onboardReq struct { // Prepare include public code path for call request handler of OIDCController func (oc *OIDCController) Prepare() { - if mode, _ := config.AuthMode(); mode != common.OIDCAuth { + if mode, _ := config.AuthMode(orm.Context()); mode != common.OIDCAuth { oc.SendPreconditionFailedError(fmt.Errorf("auth mode: %s is not OIDC based", mode)) return } @@ -121,7 +122,7 @@ func (oc *OIDCController) Callback() { } oc.SetSession(tokenKey, tokenBytes) - oidcSettings, err := config.OIDCSetting() + oidcSettings, err := config.OIDCSetting(ctx) if err != nil { oc.SendInternalServerError(err) return diff --git a/src/core/main.go b/src/core/main.go index fed76fdf2..bbb5e8ac7 100755 --- a/src/core/main.go +++ b/src/core/main.go @@ -18,6 +18,7 @@ import ( "context" "encoding/gob" "fmt" + "github.com/goharbor/harbor/src/controller/config" "net/url" "os" "os/signal" @@ -41,7 +42,6 @@ import ( _ "github.com/goharbor/harbor/src/core/auth/ldap" _ "github.com/goharbor/harbor/src/core/auth/oidc" _ "github.com/goharbor/harbor/src/core/auth/uaa" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/middlewares" "github.com/goharbor/harbor/src/core/service/token" "github.com/goharbor/harbor/src/lib/cache" @@ -189,7 +189,7 @@ func main() { if err = migration.Migrate(database); err != nil { log.Fatalf("failed to migrate: %v", err) } - if err := config.Load(); err != nil { + if err := config.Load(orm.Context()); err != nil { log.Fatalf("failed to load config: %v", err) } diff --git a/src/core/service/token/authutils.go b/src/core/service/token/authutils.go index c99fa1599..1b2f86fb6 100644 --- a/src/core/service/token/authutils.go +++ b/src/core/service/token/authutils.go @@ -17,6 +17,7 @@ package token import ( "context" "fmt" + "github.com/goharbor/harbor/src/controller/config" "strings" "time" @@ -27,7 +28,6 @@ import ( "github.com/goharbor/harbor/src/common/security" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/controller/project" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" tokenpkg "github.com/goharbor/harbor/src/pkg/token" v2 "github.com/goharbor/harbor/src/pkg/token/claims/v2" @@ -109,12 +109,12 @@ func filterAccess(ctx context.Context, access []*token.ResourceActions, } // MakeToken makes a valid jwt token based on parms. -func MakeToken(username, service string, access []*token.ResourceActions) (*models.Token, error) { +func MakeToken(ctx context.Context, username, service string, access []*token.ResourceActions) (*models.Token, error) { options, err := tokenpkg.NewOptions(signingMethod, v2.Issuer, privateKey) if err != nil { return nil, err } - expiration, err := config.TokenExpiration() + expiration, err := config.TokenExpiration(ctx) if err != nil { return nil, err } diff --git a/src/core/service/token/creator.go b/src/core/service/token/creator.go index a7aa5a86a..8b7481ba9 100644 --- a/src/core/service/token/creator.go +++ b/src/core/service/token/creator.go @@ -18,6 +18,7 @@ import ( "context" "fmt" rbac_project "github.com/goharbor/harbor/src/common/rbac/project" + "github.com/goharbor/harbor/src/controller/config" "net/http" "net/url" "strings" @@ -27,7 +28,6 @@ import ( "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/security" "github.com/goharbor/harbor/src/controller/project" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" ) @@ -245,7 +245,7 @@ func (g generalCreator) Create(r *http.Request) (*models.Token, error) { if err != nil { return nil, err } - return MakeToken(ctx.GetUsername(), g.service, access) + return MakeToken(r.Context(), ctx.GetUsername(), g.service, access) } func parseScopes(u *url.URL) []string { diff --git a/src/core/service/token/token_test.go b/src/core/service/token/token_test.go index 86f761d09..a6aa19039 100644 --- a/src/core/service/token/token_test.go +++ b/src/core/service/token/token_test.go @@ -20,6 +20,9 @@ import ( "encoding/pem" "fmt" "github.com/goharbor/harbor/src/common/rbac/project" + "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" "io/ioutil" "net/url" "os" @@ -32,11 +35,11 @@ import ( "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/security" - "github.com/goharbor/harbor/src/core/config" "github.com/stretchr/testify/assert" ) func TestMain(m *testing.M) { + test.InitDatabaseFromEnv() config.Init() InitCreators() result := m.Run() @@ -133,7 +136,7 @@ func TestMakeToken(t *testing.T) { }} svc := "harbor-registry" u := "tester" - tokenJSON, err := MakeToken(u, svc, ra) + tokenJSON, err := MakeToken(orm.Context(), u, svc, ra) if err != nil { t.Errorf("Error while making token: %v", err) } diff --git a/src/core/utils/job.go b/src/core/utils/job.go index c817ccd3a..4976c3850 100644 --- a/src/core/utils/job.go +++ b/src/core/utils/job.go @@ -17,8 +17,7 @@ package utils import ( "github.com/goharbor/harbor/src/common/job" - "github.com/goharbor/harbor/src/core/config" - + "github.com/goharbor/harbor/src/controller/config" "sync" ) diff --git a/src/jobservice/job/impl/context.go b/src/jobservice/job/impl/context.go index 365e968b9..fc0692df2 100644 --- a/src/jobservice/job/impl/context.go +++ b/src/jobservice/job/impl/context.go @@ -23,12 +23,12 @@ import ( "time" o "github.com/astaxie/beego/orm" - comcfg "github.com/goharbor/harbor/src/common/config" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/jobservice/config" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/jobservice/logger" "github.com/goharbor/harbor/src/jobservice/logger/sweeper" + libCfg "github.com/goharbor/harbor/src/lib/config" "github.com/goharbor/harbor/src/lib/orm" ) @@ -45,7 +45,7 @@ type Context struct { // other required information properties map[string]interface{} // admin server client - cfgMgr comcfg.CfgManager + cfgMgr libCfg.Manager // job life cycle tracker tracker job.Tracker // job logger configs settings map lock @@ -53,10 +53,10 @@ type Context struct { } // NewContext ... -func NewContext(sysCtx context.Context, cfgMgr *comcfg.CfgManager) *Context { +func NewContext(sysCtx context.Context, cfgMgr libCfg.Manager) *Context { return &Context{ - sysContext: comcfg.NewContext(sysCtx, cfgMgr), - cfgMgr: *cfgMgr, + sysContext: libCfg.NewContext(sysCtx, cfgMgr), + cfgMgr: cfgMgr, properties: make(map[string]interface{}), } } @@ -70,7 +70,7 @@ func (c *Context) Init() error { for counter == 0 || err != nil { counter++ - err = c.cfgMgr.Load() + err = c.cfgMgr.Load(c.sysContext) if err != nil { logger.Errorf("Job context initialization error: %s\n", err.Error()) if counter < maxRetryTimes { @@ -117,12 +117,12 @@ func (c *Context) Build(tracker job.Tracker) (job.Context, error) { } // Refresh config properties - err := c.cfgMgr.Load() + err := c.cfgMgr.Load(c.sysContext) if err != nil { return nil, err } - props := c.cfgMgr.GetAll() + props := c.cfgMgr.GetAll(c.sysContext) for k, v := range props { jContext.properties[k] = v } diff --git a/src/jobservice/job/impl/context_test.go b/src/jobservice/job/impl/context_test.go index a1093c985..7706a9f3c 100644 --- a/src/jobservice/job/impl/context_test.go +++ b/src/jobservice/job/impl/context_test.go @@ -16,17 +16,20 @@ package impl import ( "context" + "github.com/goharbor/harbor/src/common" common_dao "github.com/goharbor/harbor/src/common/dao" + libCfg "github.com/goharbor/harbor/src/lib/config" "os" "testing" "github.com/goharbor/harbor/src/jobservice/common/list" - comcfg "github.com/goharbor/harbor/src/common/config" "github.com/goharbor/harbor/src/jobservice/common/utils" "github.com/goharbor/harbor/src/jobservice/config" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/jobservice/tests" + _ "github.com/goharbor/harbor/src/pkg/config/inmemory" + _ "github.com/goharbor/harbor/src/pkg/config/rest" "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -110,9 +113,11 @@ func (suite *ContextImplTestSuite) TearDownSuite() { // TestContextImpl tests the context impl func (suite *ContextImplTestSuite) TestContextImpl() { - cfgMem := comcfg.NewInMemoryManager() - cfgMem.Set("read_only", "true") - ctx := NewContext(context.Background(), cfgMem) + cfgMem, err := libCfg.GetManager(common.InMemoryCfgManager) + assert.Nil(suite.T(), err) + cont := context.Background() + cfgMem.Set(cont, "read_only", "true") + ctx := NewContext(cont, cfgMem) jCtx, err := ctx.Build(suite.tracker) require.NoErrorf(suite.T(), err, "build job context: nil error expected but got %s", err) diff --git a/src/jobservice/job/impl/gcreadonly/garbage_collection.go b/src/jobservice/job/impl/gcreadonly/garbage_collection.go index 55c8cdc43..c3b5df108 100644 --- a/src/jobservice/job/impl/gcreadonly/garbage_collection.go +++ b/src/jobservice/job/impl/gcreadonly/garbage_collection.go @@ -16,11 +16,12 @@ package gcreadonly import ( "fmt" + "github.com/goharbor/harbor/src/lib/config" + "github.com/goharbor/harbor/src/lib/orm" "os" "time" "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/config" "github.com/goharbor/harbor/src/common/registryctl" "github.com/goharbor/harbor/src/controller/artifact" "github.com/goharbor/harbor/src/controller/project" @@ -36,19 +37,21 @@ import ( var ( regCtlInit = registryctl.Init - getReadOnly = func(cfgMgr *config.CfgManager) (bool, error) { - if err := cfgMgr.Load(); err != nil { + getReadOnly = func(cfgMgr config.Manager) (bool, error) { + cxt := orm.Context() + if err := cfgMgr.Load(cxt); err != nil { return false, err } - return cfgMgr.Get(common.ReadOnly).GetBool(), nil + return cfgMgr.Get(cxt, common.ReadOnly).GetBool(), nil } - setReadOnly = func(cfgMgr *config.CfgManager, switcher bool) error { + setReadOnly = func(cfgMgr config.Manager, switcher bool) error { + cxt := orm.Context() cfg := map[string]interface{}{ common.ReadOnly: switcher, } - cfgMgr.UpdateConfig(cfg) - return cfgMgr.Save() + cfgMgr.UpdateConfig(cxt, cfg) + return cfgMgr.Save(cxt) } ) @@ -69,7 +72,7 @@ type GarbageCollector struct { projectCtl project.Controller registryCtlClient client.Client logger logger.Interface - cfgMgr *config.CfgManager + cfgMgr config.Manager CoreURL string redisURL string deleteUntagged bool @@ -164,15 +167,11 @@ func (gc *GarbageCollector) init(ctx job.Context, params job.Parameters) error { return err } - errTpl := "failed to get required property: %s" - if v, ok := ctx.Get(common.CoreURL); ok && len(v.(string)) > 0 { - gc.CoreURL = v.(string) - } else { - return fmt.Errorf(errTpl, common.CoreURL) + mgr, err := config.GetManager(common.RestCfgManager) + if err != nil { + return err } - secret := os.Getenv("JOBSERVICE_SECRET") - configURL := gc.CoreURL + common.CoreConfigPath - gc.cfgMgr = config.NewRESTCfgManager(configURL, secret) + gc.cfgMgr = mgr gc.redisURL = params["redis_url_reg"].(string) // default is to delete the untagged artifact diff --git a/src/jobservice/job/impl/gcreadonly/garbage_collection_test.go b/src/jobservice/job/impl/gcreadonly/garbage_collection_test.go index f04ecb582..bd064a892 100644 --- a/src/jobservice/job/impl/gcreadonly/garbage_collection_test.go +++ b/src/jobservice/job/impl/gcreadonly/garbage_collection_test.go @@ -15,10 +15,11 @@ package gcreadonly import ( + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/lib/config" "os" "testing" - "github.com/goharbor/harbor/src/common/config" "github.com/goharbor/harbor/src/common/models" commom_regctl "github.com/goharbor/harbor/src/common/registryctl" "github.com/goharbor/harbor/src/controller/project" @@ -26,6 +27,10 @@ import ( "github.com/goharbor/harbor/src/pkg/artifact" "github.com/goharbor/harbor/src/pkg/artifactrash/model" pkg_blob "github.com/goharbor/harbor/src/pkg/blob/models" + _ "github.com/goharbor/harbor/src/pkg/config/db" + _ "github.com/goharbor/harbor/src/pkg/config/inmemory" + _ "github.com/goharbor/harbor/src/pkg/config/rest" + artifacttesting "github.com/goharbor/harbor/src/testing/controller/artifact" projecttesting "github.com/goharbor/harbor/src/testing/controller/project" mockjobservice "github.com/goharbor/harbor/src/testing/jobservice" @@ -47,8 +52,8 @@ type gcTestSuite struct { originalProjectCtl project.Controller regCtlInit func() - setReadOnly func(cfgMgr *config.CfgManager, switcher bool) error - getReadOnly func(cfgMgr *config.CfgManager) (bool, error) + setReadOnly func(cfgMgr config.Manager, switcher bool) error + getReadOnly func(cfgMgr config.Manager) (bool, error) } func (suite *gcTestSuite) SetupTest() { @@ -62,8 +67,8 @@ func (suite *gcTestSuite) SetupTest() { project.Ctl = suite.projectCtl regCtlInit = func() { commom_regctl.RegistryCtlClient = suite.registryCtlClient } - setReadOnly = func(cfgMgr *config.CfgManager, switcher bool) error { return nil } - getReadOnly = func(cfgMgr *config.CfgManager) (bool, error) { return true, nil } + setReadOnly = func(cfgMgr config.Manager, switcher bool) error { return nil } + getReadOnly = func(cfgMgr config.Manager) (bool, error) { return true, nil } } func (suite *gcTestSuite) TearDownTest() { @@ -237,11 +242,12 @@ func (suite *gcTestSuite) TestRun() { }, nil) mock.OnAnything(suite.blobMgr, "CleanupAssociationsForProject").Return(nil) - + mgr, err := config.GetManager(common.InMemoryCfgManager) + suite.Nil(err) gc := &GarbageCollector{ artCtl: suite.artifactCtl, artrashMgr: suite.artrashMgr, - cfgMgr: config.NewInMemoryManager(), + cfgMgr: mgr, blobMgr: suite.blobMgr, registryCtlClient: suite.registryCtlClient, } diff --git a/src/jobservice/main.go b/src/jobservice/main.go index ca146b102..4467c1ae4 100644 --- a/src/jobservice/main.go +++ b/src/jobservice/main.go @@ -19,15 +19,15 @@ import ( "errors" "flag" "fmt" - "github.com/goharbor/harbor/src/common" - comcfg "github.com/goharbor/harbor/src/common/config" "github.com/goharbor/harbor/src/jobservice/common/utils" "github.com/goharbor/harbor/src/jobservice/config" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/jobservice/job/impl" "github.com/goharbor/harbor/src/jobservice/logger" "github.com/goharbor/harbor/src/jobservice/runtime" + cfgLib "github.com/goharbor/harbor/src/lib/config" + _ "github.com/goharbor/harbor/src/pkg/config/rest" ) func main() { @@ -63,9 +63,10 @@ func main() { if utils.IsEmptyStr(secret) { return nil, errors.New("empty auth secret") } - coreURL := config.GetCoreURL() - configURL := coreURL + common.CoreConfigPath - cfgMgr := comcfg.NewRESTCfgManager(configURL, secret) + cfgMgr, err := cfgLib.GetManager(common.RestCfgManager) + if err != nil { + return nil, err + } jobCtx := impl.NewContext(ctx, cfgMgr) if err := jobCtx.Init(); err != nil { diff --git a/src/lib/config/config.go b/src/lib/config/config.go new file mode 100644 index 000000000..77af92cfc --- /dev/null +++ b/src/lib/config/config.go @@ -0,0 +1,48 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "errors" + "github.com/goharbor/harbor/src/lib/log" + "sync" +) + +var ( + managersMU sync.RWMutex + managers = make(map[string]Manager) +) + +// Register register the config manager +func Register(name string, mgr Manager) { + managersMU.Lock() + defer managersMU.Unlock() + if mgr == nil { + log.Error("Register manager is nil") + } + if _, dup := managers[name]; dup { + log.Errorf("Register called twice for manager " + name) + } + managers[name] = mgr +} + +// GetManager get the configure manager by name +func GetManager(name string) (Manager, error) { + mgr, ok := managers[name] + if !ok { + return nil, errors.New("config manager is not registered: " + name) + } + return mgr, nil +} diff --git a/src/common/config/context.go b/src/lib/config/context.go similarity index 50% rename from src/common/config/context.go rename to src/lib/config/context.go index f75a1f0e7..b14da4591 100644 --- a/src/common/config/context.go +++ b/src/lib/config/context.go @@ -16,17 +16,32 @@ package config import ( "context" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/lib/config/metadata" ) type cfgMgrKey struct{} +// Manager defines the operation for config +type Manager interface { + Load(ctx context.Context) error + Set(ctx context.Context, key string, value interface{}) + Save(ctx context.Context) error + Get(ctx context.Context, key string) *metadata.ConfigureValue + UpdateConfig(ctx context.Context, cfgs map[string]interface{}) error + GetUserCfgs(ctx context.Context) map[string]interface{} + ValidateCfg(ctx context.Context, cfgs map[string]interface{}) error + GetAll(ctx context.Context) map[string]interface{} + GetDatabaseCfg() *models.Database +} + // FromContext returns CfgManager from context -func FromContext(ctx context.Context) (*CfgManager, bool) { - m, ok := ctx.Value(cfgMgrKey{}).(*CfgManager) +func FromContext(ctx context.Context) (Manager, bool) { + m, ok := ctx.Value(cfgMgrKey{}).(Manager) return m, ok } // NewContext returns context with CfgManager -func NewContext(ctx context.Context, m *CfgManager) context.Context { +func NewContext(ctx context.Context, m Manager) context.Context { return context.WithValue(ctx, cfgMgrKey{}, m) } diff --git a/src/common/config/metadata/metadata.go b/src/lib/config/metadata/metadata.go similarity index 100% rename from src/common/config/metadata/metadata.go rename to src/lib/config/metadata/metadata.go diff --git a/src/common/config/metadata/metadata_test.go b/src/lib/config/metadata/metadata_test.go similarity index 100% rename from src/common/config/metadata/metadata_test.go rename to src/lib/config/metadata/metadata_test.go diff --git a/src/common/config/metadata/metadatalist.go b/src/lib/config/metadata/metadatalist.go similarity index 70% rename from src/common/config/metadata/metadatalist.go rename to src/lib/config/metadata/metadatalist.go index 4ded1d700..54062e839 100644 --- a/src/common/config/metadata/metadatalist.go +++ b/src/lib/config/metadata/metadatalist.go @@ -30,8 +30,10 @@ type Item struct { Name string `json:"name,omitempty"` // It can be &IntType{}, &StringType{}, &BoolType{}, &PasswordType{}, &MapType{} etc, any type interface implementation ItemType Type - // TODO: Clarify the usage of this attribute + // Editable means it can updated by configure api, For system configure, the editable is always false, for user configure, it may depends Editable bool `json:"editable,omitempty"` + // Description - Describle the usage of the configure item + Description string } // Constant for configure item @@ -62,7 +64,7 @@ var ( ConfigList = []Item{ {Name: common.AdminInitialPassword, Scope: SystemScope, Group: BasicGroup, EnvKey: "HARBOR_ADMIN_PASSWORD", DefaultValue: "", ItemType: &PasswordType{}, Editable: true}, - {Name: common.AUTHMode, Scope: UserScope, Group: BasicGroup, EnvKey: "AUTH_MODE", DefaultValue: "db_auth", ItemType: &AuthModeType{}, Editable: false}, + {Name: common.AUTHMode, Scope: UserScope, Group: BasicGroup, EnvKey: "AUTH_MODE", DefaultValue: "db_auth", ItemType: &AuthModeType{}, Editable: false, Description: `The auth mode of current system, such as "db_auth", "ldap_auth", "oidc_auth"`}, {Name: common.ChartRepoURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "CHART_REPOSITORY_URL", DefaultValue: "http://chartmuseum:9999", ItemType: &StringType{}, Editable: false}, {Name: common.TrivyAdapterURL, Scope: SystemScope, Group: TrivyGroup, EnvKey: "TRIVY_ADAPTER_URL", DefaultValue: "http://trivy-adapter:8080", ItemType: &StringType{}, Editable: false}, @@ -71,37 +73,37 @@ var ( {Name: common.CoreLocalURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "CORE_LOCAL_URL", DefaultValue: "http://127.0.0.1:8080", ItemType: &StringType{}, Editable: false}, {Name: common.DatabaseType, Scope: SystemScope, Group: BasicGroup, EnvKey: "DATABASE_TYPE", DefaultValue: "postgresql", ItemType: &StringType{}, Editable: false}, - {Name: common.EmailFrom, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_FROM", DefaultValue: "admin ", ItemType: &StringType{}, Editable: false}, - {Name: common.EmailHost, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_HOST", DefaultValue: "smtp.mydomain.com", ItemType: &StringType{}, Editable: false}, - {Name: common.EmailIdentity, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_IDENTITY", DefaultValue: "", ItemType: &StringType{}, Editable: false}, - {Name: common.EmailInsecure, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_INSECURE", DefaultValue: "false", ItemType: &BoolType{}, Editable: false}, - {Name: common.EmailPassword, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_PWD", DefaultValue: "", ItemType: &PasswordType{}, Editable: false}, - {Name: common.EmailPort, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_PORT", DefaultValue: "25", ItemType: &PortType{}, Editable: false}, - {Name: common.EmailSSL, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_SSL", DefaultValue: "false", ItemType: &BoolType{}, Editable: false}, - {Name: common.EmailUsername, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_USR", DefaultValue: "sample_admin@mydomain.com", ItemType: &StringType{}, Editable: false}, + {Name: common.EmailFrom, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_FROM", DefaultValue: "admin ", ItemType: &StringType{}, Editable: false, Description: `The sender name for Email notification.`}, + {Name: common.EmailHost, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_HOST", DefaultValue: "smtp.mydomain.com", ItemType: &StringType{}, Editable: false, Description: `The hostname of SMTP server that sends Email notification.`}, + {Name: common.EmailIdentity, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_IDENTITY", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `By default it's empty so the email_username is picked`}, + {Name: common.EmailInsecure, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_INSECURE", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `Whether or not the certificate will be verified when Harbor tries to access the email server.`}, + {Name: common.EmailPassword, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_PWD", DefaultValue: "", ItemType: &PasswordType{}, Editable: false, Description: `Email password`}, + {Name: common.EmailPort, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_PORT", DefaultValue: "25", ItemType: &PortType{}, Editable: false, Description: `The port of SMTP server`}, + {Name: common.EmailSSL, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_SSL", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `When it''s set to true the system will access Email server via TLS by default. If it''s set to false, it still will handle "STARTTLS" from server side.`}, + {Name: common.EmailUsername, Scope: UserScope, Group: EmailGroup, EnvKey: "EMAIL_USR", DefaultValue: "sample_admin@mydomain.com", ItemType: &StringType{}, Editable: false, Description: `The username for authenticate against SMTP server`}, {Name: common.ExtEndpoint, Scope: SystemScope, Group: BasicGroup, EnvKey: "EXT_ENDPOINT", DefaultValue: "https://host01.com", ItemType: &StringType{}, Editable: false}, {Name: common.JobServiceURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "JOBSERVICE_URL", DefaultValue: "http://jobservice:8080", ItemType: &StringType{}, Editable: false}, - {Name: common.LDAPBaseDN, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_BASE_DN", DefaultValue: "", ItemType: &NonEmptyStringType{}, Editable: false}, - {Name: common.LDAPFilter, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_FILTER", DefaultValue: "", ItemType: &StringType{}, Editable: false}, - {Name: common.LDAPGroupBaseDN, Scope: UserScope, Group: LdapGroupGroup, EnvKey: "LDAP_GROUP_BASE_DN", DefaultValue: "", ItemType: &StringType{}, Editable: false}, - {Name: common.LDAPGroupAdminDn, Scope: UserScope, Group: LdapGroupGroup, EnvKey: "LDAP_GROUP_ADMIN_DN", DefaultValue: "", ItemType: &StringType{}, Editable: false}, - {Name: common.LDAPGroupAttributeName, Scope: UserScope, Group: LdapGroupGroup, EnvKey: "LDAP_GROUP_GID", DefaultValue: "", ItemType: &StringType{}, Editable: false}, - {Name: common.LDAPGroupSearchFilter, Scope: UserScope, Group: LdapGroupGroup, EnvKey: "LDAP_GROUP_FILTER", DefaultValue: "", ItemType: &StringType{}, Editable: false}, - {Name: common.LDAPGroupSearchScope, Scope: UserScope, Group: LdapGroupGroup, EnvKey: "LDAP_GROUP_SCOPE", DefaultValue: "2", ItemType: &LdapScopeType{}, Editable: false}, - {Name: common.LDAPScope, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_SCOPE", DefaultValue: "2", ItemType: &LdapScopeType{}, Editable: false}, - {Name: common.LDAPSearchDN, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_SEARCH_DN", DefaultValue: "", ItemType: &StringType{}, Editable: false}, - {Name: common.LDAPSearchPwd, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_SEARCH_PWD", DefaultValue: "", ItemType: &PasswordType{}, Editable: false}, - {Name: common.LDAPTimeout, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_TIMEOUT", DefaultValue: "5", ItemType: &IntType{}, Editable: false}, - {Name: common.LDAPUID, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_UID", DefaultValue: "cn", ItemType: &NonEmptyStringType{}, Editable: false}, - {Name: common.LDAPURL, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_URL", DefaultValue: "", ItemType: &NonEmptyStringType{}, Editable: false}, - {Name: common.LDAPVerifyCert, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_VERIFY_CERT", DefaultValue: "true", ItemType: &BoolType{}, Editable: false}, - {Name: common.LDAPGroupMembershipAttribute, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_GROUP_MEMBERSHIP_ATTRIBUTE", DefaultValue: "memberof", ItemType: &StringType{}, Editable: true}, + {Name: common.LDAPBaseDN, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_BASE_DN", DefaultValue: "", ItemType: &NonEmptyStringType{}, Editable: false, Description: `The Base DN for LDAP binding.`}, + {Name: common.LDAPFilter, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_FILTER", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The filter for LDAP search`}, + {Name: common.LDAPGroupBaseDN, Scope: UserScope, Group: LdapGroupGroup, EnvKey: "LDAP_GROUP_BASE_DN", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The base DN to search LDAP group.`}, + {Name: common.LDAPGroupAdminDn, Scope: UserScope, Group: LdapGroupGroup, EnvKey: "LDAP_GROUP_ADMIN_DN", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `Specify the ldap group which have the same privilege with Harbor admin`}, + {Name: common.LDAPGroupAttributeName, Scope: UserScope, Group: LdapGroupGroup, EnvKey: "LDAP_GROUP_GID", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The attribute which is used as identity of the LDAP group, default is cn.'`}, + {Name: common.LDAPGroupSearchFilter, Scope: UserScope, Group: LdapGroupGroup, EnvKey: "LDAP_GROUP_FILTER", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The filter to search the ldap group`}, + {Name: common.LDAPGroupSearchScope, Scope: UserScope, Group: LdapGroupGroup, EnvKey: "LDAP_GROUP_SCOPE", DefaultValue: "2", ItemType: &LdapScopeType{}, Editable: false, Description: `The scope to search ldap group. ''0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE''`}, + {Name: common.LDAPScope, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_SCOPE", DefaultValue: "2", ItemType: &LdapScopeType{}, Editable: false, Description: `The scope to search ldap users,'0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE'`}, + {Name: common.LDAPSearchDN, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_SEARCH_DN", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The DN of the user to do the search.`}, + {Name: common.LDAPSearchPwd, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_SEARCH_PWD", DefaultValue: "", ItemType: &PasswordType{}, Editable: false, Description: `The password of the ldap search dn`}, + {Name: common.LDAPTimeout, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_TIMEOUT", DefaultValue: "5", ItemType: &IntType{}, Editable: false, Description: `Timeout in seconds for connection to LDAP server`}, + {Name: common.LDAPUID, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_UID", DefaultValue: "cn", ItemType: &NonEmptyStringType{}, Editable: false, Description: `The attribute which is used as identity for the LDAP binding, such as "CN" or "SAMAccountname"`}, + {Name: common.LDAPURL, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_URL", DefaultValue: "", ItemType: &NonEmptyStringType{}, Editable: false, Description: `The URL of LDAP server`}, + {Name: common.LDAPVerifyCert, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_VERIFY_CERT", DefaultValue: "true", ItemType: &BoolType{}, Editable: false, Description: `Whether verify your OIDC server certificate, disable it if your OIDC server is hosted via self-hosted certificate.`}, + {Name: common.LDAPGroupMembershipAttribute, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_GROUP_MEMBERSHIP_ATTRIBUTE", DefaultValue: "memberof", ItemType: &StringType{}, Editable: true, Description: `The user attribute to identify the group membership`}, {Name: common.MaxJobWorkers, Scope: SystemScope, Group: BasicGroup, EnvKey: "MAX_JOB_WORKERS", DefaultValue: "10", ItemType: &IntType{}, Editable: false}, {Name: common.NotaryURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "NOTARY_URL", DefaultValue: "http://notary-server:4443", ItemType: &StringType{}, Editable: false}, - {Name: common.ScanAllPolicy, Scope: UserScope, Group: BasicGroup, EnvKey: "", DefaultValue: "", ItemType: &MapType{}, Editable: false}, + {Name: common.ScanAllPolicy, Scope: UserScope, Group: BasicGroup, EnvKey: "", DefaultValue: "", ItemType: &MapType{}, Editable: false, Description: `The policy to scan images`}, {Name: common.PostGreSQLDatabase, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_DATABASE", DefaultValue: "registry", ItemType: &StringType{}, Editable: false}, {Name: common.PostGreSQLHOST, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_HOST", DefaultValue: "postgresql", ItemType: &StringType{}, Editable: false}, @@ -112,53 +114,53 @@ var ( {Name: common.PostGreSQLMaxIdleConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_IDLE_CONNS", DefaultValue: "2", ItemType: &IntType{}, Editable: false}, {Name: common.PostGreSQLMaxOpenConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_OPEN_CONNS", DefaultValue: "0", ItemType: &IntType{}, Editable: false}, - {Name: common.ProjectCreationRestriction, Scope: UserScope, Group: BasicGroup, EnvKey: "PROJECT_CREATION_RESTRICTION", DefaultValue: common.ProCrtRestrEveryone, ItemType: &ProjectCreationRestrictionType{}, Editable: false}, - {Name: common.ReadOnly, Scope: UserScope, Group: BasicGroup, EnvKey: "READ_ONLY", DefaultValue: "false", ItemType: &BoolType{}, Editable: false}, + {Name: common.ProjectCreationRestriction, Scope: UserScope, Group: BasicGroup, EnvKey: "PROJECT_CREATION_RESTRICTION", DefaultValue: common.ProCrtRestrEveryone, ItemType: &ProjectCreationRestrictionType{}, Editable: false, Description: `Indicate who can create projects, it could be ''adminonly'' or ''everyone''.`}, + {Name: common.ReadOnly, Scope: UserScope, Group: BasicGroup, EnvKey: "READ_ONLY", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `The flag to indicate whether Harbor is in readonly mode.`}, {Name: common.RegistryStorageProviderName, Scope: SystemScope, Group: BasicGroup, EnvKey: "REGISTRY_STORAGE_PROVIDER_NAME", DefaultValue: "filesystem", ItemType: &StringType{}, Editable: false}, {Name: common.RegistryURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "REGISTRY_URL", DefaultValue: "http://registry:5000", ItemType: &StringType{}, Editable: false}, {Name: common.RegistryControllerURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "REGISTRY_CONTROLLER_URL", DefaultValue: "http://registryctl:8080", ItemType: &StringType{}, Editable: false}, - {Name: common.SelfRegistration, Scope: UserScope, Group: BasicGroup, EnvKey: "SELF_REGISTRATION", DefaultValue: "false", ItemType: &BoolType{}, Editable: false}, - {Name: common.TokenExpiration, Scope: UserScope, Group: BasicGroup, EnvKey: "TOKEN_EXPIRATION", DefaultValue: "30", ItemType: &IntType{}, Editable: false}, + {Name: common.SelfRegistration, Scope: UserScope, Group: BasicGroup, EnvKey: "SELF_REGISTRATION", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `Whether the Harbor instance supports self-registration. If it''s set to false, admin need to add user to the instance.`}, + {Name: common.TokenExpiration, Scope: UserScope, Group: BasicGroup, EnvKey: "TOKEN_EXPIRATION", DefaultValue: "30", ItemType: &IntType{}, Editable: false, Description: `The expiration time of the token for internal Registry, in minutes.`}, {Name: common.TokenServiceURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "TOKEN_SERVICE_URL", DefaultValue: "http://core:8080/service/token", ItemType: &StringType{}, Editable: false}, - {Name: common.UAAClientID, Scope: UserScope, Group: UAAGroup, EnvKey: "UAA_CLIENTID", DefaultValue: "", ItemType: &StringType{}, Editable: false}, - {Name: common.UAAClientSecret, Scope: UserScope, Group: UAAGroup, EnvKey: "UAA_CLIENTSECRET", DefaultValue: "", ItemType: &StringType{}, Editable: false}, - {Name: common.UAAEndpoint, Scope: UserScope, Group: UAAGroup, EnvKey: "UAA_ENDPOINT", DefaultValue: "", ItemType: &StringType{}, Editable: false}, - {Name: common.UAAVerifyCert, Scope: UserScope, Group: UAAGroup, EnvKey: "UAA_VERIFY_CERT", DefaultValue: "false", ItemType: &BoolType{}, Editable: false}, + {Name: common.UAAClientID, Scope: UserScope, Group: UAAGroup, EnvKey: "UAA_CLIENTID", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The client id of UAA`}, + {Name: common.UAAClientSecret, Scope: UserScope, Group: UAAGroup, EnvKey: "UAA_CLIENTSECRET", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The client secret of the UAA`}, + {Name: common.UAAEndpoint, Scope: UserScope, Group: UAAGroup, EnvKey: "UAA_ENDPOINT", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The endpoint of the UAA`}, + {Name: common.UAAVerifyCert, Scope: UserScope, Group: UAAGroup, EnvKey: "UAA_VERIFY_CERT", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `Verify the certificate in UAA server`}, - {Name: common.HTTPAuthProxyEndpoint, Scope: UserScope, Group: HTTPAuthGroup, ItemType: &StringType{}}, - {Name: common.HTTPAuthProxyTokenReviewEndpoint, Scope: UserScope, Group: HTTPAuthGroup, ItemType: &StringType{}}, - {Name: common.HTTPAuthProxyAdminGroups, Scope: UserScope, Group: HTTPAuthGroup, ItemType: &StringType{}}, - {Name: common.HTTPAuthProxyAdminUsernames, Scope: UserScope, Group: HTTPAuthGroup, ItemType: &StringType{}}, - {Name: common.HTTPAuthProxyVerifyCert, Scope: UserScope, Group: HTTPAuthGroup, DefaultValue: "true", ItemType: &BoolType{}}, - {Name: common.HTTPAuthProxySkipSearch, Scope: UserScope, Group: HTTPAuthGroup, DefaultValue: "false", ItemType: &BoolType{}}, - {Name: common.HTTPAuthProxyServerCertificate, Scope: UserScope, Group: HTTPAuthGroup, ItemType: &StringType{}}, + {Name: common.HTTPAuthProxyEndpoint, Scope: UserScope, Group: HTTPAuthGroup, ItemType: &StringType{}, Description: `The endpoint of the HTTP auth`}, + {Name: common.HTTPAuthProxyTokenReviewEndpoint, Scope: UserScope, Group: HTTPAuthGroup, ItemType: &StringType{}, Description: `The token review endpoint`}, + {Name: common.HTTPAuthProxyAdminGroups, Scope: UserScope, Group: HTTPAuthGroup, ItemType: &StringType{}, Description: `The group which has the harbor admin privileges`}, + {Name: common.HTTPAuthProxyAdminUsernames, Scope: UserScope, Group: HTTPAuthGroup, ItemType: &StringType{}, Description: `The username which has the harbor admin privileges`}, + {Name: common.HTTPAuthProxyVerifyCert, Scope: UserScope, Group: HTTPAuthGroup, DefaultValue: "true", ItemType: &BoolType{}, Description: `Verify the HTTP auth provider's certificate`}, + {Name: common.HTTPAuthProxySkipSearch, Scope: UserScope, Group: HTTPAuthGroup, DefaultValue: "false", ItemType: &BoolType{}, Description: `Search user before onboard`}, + {Name: common.HTTPAuthProxyServerCertificate, Scope: UserScope, Group: HTTPAuthGroup, ItemType: &StringType{}, Description: `The certificate of the HTTP auth provider`}, - {Name: common.OIDCName, Scope: UserScope, Group: OIDCGroup, ItemType: &StringType{}}, - {Name: common.OIDCEndpoint, Scope: UserScope, Group: OIDCGroup, ItemType: &StringType{}}, - {Name: common.OIDCCLientID, Scope: UserScope, Group: OIDCGroup, ItemType: &StringType{}}, - {Name: common.OIDCClientSecret, Scope: UserScope, Group: OIDCGroup, ItemType: &PasswordType{}}, - {Name: common.OIDCGroupsClaim, Scope: UserScope, Group: OIDCGroup, ItemType: &StringType{}}, - {Name: common.OIDCAdminGroup, Scope: UserScope, Group: OIDCGroup, ItemType: &StringType{}}, - {Name: common.OIDCScope, Scope: UserScope, Group: OIDCGroup, ItemType: &StringType{}}, - {Name: common.OIDCUserClaim, Scope: UserScope, Group: OIDCGroup, ItemType: &StringType{}}, - {Name: common.OIDCVerifyCert, Scope: UserScope, Group: OIDCGroup, DefaultValue: "true", ItemType: &BoolType{}}, - {Name: common.OIDCAutoOnboard, Scope: UserScope, Group: OIDCGroup, DefaultValue: "false", ItemType: &BoolType{}}, - {Name: common.OIDCExtraRedirectParms, Scope: UserScope, Group: OIDCGroup, DefaultValue: "{}", ItemType: &StringToStringMapType{}}, + {Name: common.OIDCName, Scope: UserScope, Group: OIDCGroup, ItemType: &StringType{}, Description: `The OIDC provider name`}, + {Name: common.OIDCEndpoint, Scope: UserScope, Group: OIDCGroup, ItemType: &StringType{}, Description: `The endpoint of the OIDC provider`}, + {Name: common.OIDCCLientID, Scope: UserScope, Group: OIDCGroup, ItemType: &StringType{}, Description: `The client ID of the OIDC provider`}, + {Name: common.OIDCClientSecret, Scope: UserScope, Group: OIDCGroup, ItemType: &PasswordType{}, Description: `The OIDC provider secret`}, + {Name: common.OIDCGroupsClaim, Scope: UserScope, Group: OIDCGroup, ItemType: &StringType{}, Description: `The attribute claims the group name`}, + {Name: common.OIDCAdminGroup, Scope: UserScope, Group: OIDCGroup, ItemType: &StringType{}, Description: `The OIDC group which has the harbor admin privileges`}, + {Name: common.OIDCScope, Scope: UserScope, Group: OIDCGroup, ItemType: &StringType{}, Description: `The scope of the OIDC provider`}, + {Name: common.OIDCUserClaim, Scope: UserScope, Group: OIDCGroup, ItemType: &StringType{}, Description: `The attribute claims the username`}, + {Name: common.OIDCVerifyCert, Scope: UserScope, Group: OIDCGroup, DefaultValue: "true", ItemType: &BoolType{}, Description: `Verify the OIDC provider's certificate'`}, + {Name: common.OIDCAutoOnboard, Scope: UserScope, Group: OIDCGroup, DefaultValue: "false", ItemType: &BoolType{}, Description: `Auto onboard the OIDC user`}, + {Name: common.OIDCExtraRedirectParms, Scope: UserScope, Group: OIDCGroup, DefaultValue: "{}", ItemType: &StringToStringMapType{}, Description: `Extra parameters to add when redirect request to OIDC provider`}, {Name: common.WithChartMuseum, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_CHARTMUSEUM", DefaultValue: "false", ItemType: &BoolType{}, Editable: true}, {Name: common.WithTrivy, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_TRIVY", DefaultValue: "false", ItemType: &BoolType{}, Editable: true}, {Name: common.WithNotary, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_NOTARY", DefaultValue: "false", ItemType: &BoolType{}, Editable: true}, // the unit of expiration is days - {Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "30", ItemType: &IntType{}, Editable: true}, - {Name: common.RobotNamePrefix, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_NAME_PREFIX", DefaultValue: "robot$", ItemType: &StringType{}, Editable: true}, - {Name: common.NotificationEnable, Scope: UserScope, Group: BasicGroup, EnvKey: "NOTIFICATION_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true}, + {Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "30", ItemType: &IntType{}, Editable: true, Description: `The robot account token duration in days`}, + {Name: common.RobotNamePrefix, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_NAME_PREFIX", DefaultValue: "robot$", ItemType: &StringType{}, Editable: true, Description: `The rebot account name prefix`}, + {Name: common.NotificationEnable, Scope: UserScope, Group: BasicGroup, EnvKey: "NOTIFICATION_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true, Description: `Enable notification`}, {Name: common.MetricEnable, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_ENABLE", DefaultValue: "false", ItemType: &BoolType{}, Editable: true}, {Name: common.MetricPort, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_PORT", DefaultValue: "9090", ItemType: &IntType{}, Editable: true}, {Name: common.MetricPath, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_PATH", DefaultValue: "/metrics", ItemType: &StringType{}, Editable: true}, - {Name: common.QuotaPerProjectEnable, Scope: UserScope, Group: QuotaGroup, EnvKey: "QUOTA_PER_PROJECT_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true}, - {Name: common.StoragePerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "STORAGE_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true}, + {Name: common.QuotaPerProjectEnable, Scope: UserScope, Group: QuotaGroup, EnvKey: "QUOTA_PER_PROJECT_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true, Description: `Enable quota per project`}, + {Name: common.StoragePerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "STORAGE_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true, Description: `The storage quota per project`}, } ) diff --git a/src/common/config/metadata/type.go b/src/lib/config/metadata/type.go similarity index 100% rename from src/common/config/metadata/type.go rename to src/lib/config/metadata/type.go diff --git a/src/common/config/metadata/type_test.go b/src/lib/config/metadata/type_test.go similarity index 100% rename from src/common/config/metadata/type_test.go rename to src/lib/config/metadata/type_test.go diff --git a/src/common/config/metadata/value.go b/src/lib/config/metadata/value.go similarity index 100% rename from src/common/config/metadata/value.go rename to src/lib/config/metadata/value.go diff --git a/src/common/config/metadata/value_test.go b/src/lib/config/metadata/value_test.go similarity index 100% rename from src/common/config/metadata/value_test.go rename to src/lib/config/metadata/value_test.go diff --git a/src/lib/config/metadata/yaml/genyaml.go b/src/lib/config/metadata/yaml/genyaml.go new file mode 100644 index 000000000..0cd86f9d1 --- /dev/null +++ b/src/lib/config/metadata/yaml/genyaml.go @@ -0,0 +1,127 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/goharbor/harbor/src/lib/config/metadata" + "os" + "text/template" +) + +const cfgTemplate = ` Configuration: + type: object + properties: {{ range .Items }} + {{ .Name }}: + type: {{ .Type }} + description: {{ .Description }} {{ end }} +` + +const responseTemplate = ` ConfigurationResponse: + type: object + properties: {{ range .Items }} + {{ .Name }}: + $ref: '#/definitions/{{ .Type }}' + description: {{ .Description }} {{ end }} +` + +type document struct { + Items []templateItem +} + +type templateItem struct { + Name string + Type string + Description string +} + +func userCfgItems(isResponse bool) []templateItem { + items := make([]templateItem, 0) + for _, i := range metadata.ConfigList { + if i.Scope == metadata.SystemScope { + continue + } + // skip scan_all_policy + if i.Name == "scan_all_policy" { + continue + } + if _, ok := i.ItemType.(*metadata.PasswordType); isResponse && ok { + continue + } + t := requestType(i.ItemType) + if isResponse { + t = toResponseType(t) + } + item := templateItem{ + Name: i.Name, + Type: t, + Description: i.Description, + } + items = append(items, item) + } + return items +} + +type yamlFile struct { + Name string + IsResponse bool + TempName string +} + +var responseMap = map[string]string{ + "string": "StringConfigItem", + "boolean": "BoolConfigItem", + "integer": "IntegerConfigItem", +} + +func main() { + l := []yamlFile{ + {"configurations.yml", false, cfgTemplate}, + {"configurationsResponse.yml", true, responseTemplate}, + } + for _, file := range l { + f, err := os.Create(file.Name) + if err != nil { + panic(err) + } + doc := document{ + Items: userCfgItems(file.IsResponse), + } + tmpl, err := template.New("test").Parse(file.TempName) + if err != nil { + panic(err) + } + err = tmpl.Execute(f, doc) + if err != nil { + panic(err) + } + f.Close() + } +} + +func requestType(item metadata.Type) string { + switch item.(type) { + case *metadata.StringType: + return "string" + case *metadata.BoolType: + return "boolean" + case *metadata.IntType, *metadata.PortType, *metadata.QuotaType, *metadata.LdapScopeType, *metadata.Int64Type: + return "integer" + } + return "string" +} + +func toResponseType(typeName string) string { + return responseMap[typeName] +} diff --git a/src/lib/config/model.go b/src/lib/config/model.go new file mode 100644 index 000000000..a486d37b7 --- /dev/null +++ b/src/lib/config/model.go @@ -0,0 +1,24 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +// Value ... +type Value struct { + Val interface{} `json:"value"` + Editable bool `json:"editable"` +} + +// InternalCfg internal configure response model +type InternalCfg map[string]*Value diff --git a/src/common/models/config.go b/src/lib/config/models/model.go similarity index 55% rename from src/common/models/config.go rename to src/lib/config/models/model.go index 73c268667..4dcc2db3e 100644 --- a/src/common/models/config.go +++ b/src/lib/config/models/model.go @@ -1,59 +1,22 @@ -// Copyright Project Harbor Authors +// Copyright Project Harbor Authors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package models -/* -// Authentication ... -type Authentication struct { - Mode string `json:"mode"` - SelfRegistration bool `json:"self_registration"` - LDAP *LDAP `json:"ldap,omitempty"` -} -*/ - -// Database ... -type Database struct { - Type string `json:"type"` - PostGreSQL *PostGreSQL `json:"postgresql,omitempty"` -} - -// MySQL ... -type MySQL struct { - Host string `json:"host"` - Port int `json:"port"` - Username string `json:"username"` - Password string `json:"password,omitempty"` - Database string `json:"database"` -} - -// SQLite ... -type SQLite struct { - File string `json:"file"` -} - -// PostGreSQL ... -type PostGreSQL struct { - Host string `json:"host"` - Port int `json:"port"` - Username string `json:"username"` - Password string `json:"password,omitempty"` - Database string `json:"database"` - SSLMode string `json:"sslmode"` - MaxIdleConns int `json:"max_idle_conns"` - MaxOpenConns int `json:"max_open_conns"` -} +import ( + "github.com/astaxie/beego/orm" +) // Email ... type Email struct { @@ -99,6 +62,10 @@ type QuotaSetting struct { StoragePerProject int64 `json:"storage_per_project"` } +func init() { + orm.RegisterModel(new(ConfigEntry)) +} + // ConfigEntry ... type ConfigEntry struct { ID int64 `orm:"pk;auto;column(id)" json:"-"` diff --git a/src/lib/orm/error.go b/src/lib/orm/error.go index c2674c3d6..7b7a9c975 100644 --- a/src/lib/orm/error.go +++ b/src/lib/orm/error.go @@ -59,7 +59,7 @@ func AsNotFoundError(err error, messageFormat string, args ...interface{}) *erro // AsConflictError checks whether the err is duplicate key error. If it it, wrap it // as a src/internal/error.Error with conflict error code, else return nil func AsConflictError(err error, messageFormat string, args ...interface{}) *errors.Error { - if isDuplicateKeyError(err) { + if IsDuplicateKeyError(err) { e := errors.New(err). WithCode(errors.ConflictCode). WithMessage(messageFormat, args...) @@ -80,7 +80,8 @@ func AsForeignKeyError(err error, messageFormat string, args ...interface{}) *er return nil } -func isDuplicateKeyError(err error) bool { +// IsDuplicateKeyError check the duplicate key error +func IsDuplicateKeyError(err error) bool { var pqErr *pq.Error if errors.As(err, &pqErr) && pqErr.Code == "23505" { return true diff --git a/src/lib/orm/orm.go b/src/lib/orm/orm.go index 3583718ff..497234617 100644 --- a/src/lib/orm/orm.go +++ b/src/lib/orm/orm.go @@ -18,7 +18,6 @@ import ( "context" "errors" "fmt" - "github.com/goharbor/harbor/src/common/dao" "strconv" "strings" @@ -158,7 +157,7 @@ func ReadOrCreate(ctx context.Context, md interface{}, col1 string, cols ...stri } // got a duplicate key error, try to read again - if isDuplicateKeyError(err) { + if IsDuplicateKeyError(err) { err = o.Read(md, cols...) } @@ -191,9 +190,12 @@ func CreateInClause(ctx context.Context, sql string, args ...interface{}) (strin return fmt.Sprintf(`IN (%s)`, strings.Join(idStrs, ",")), nil } -// Escape special characters +// Escape .. func Escape(str string) string { - return dao.Escape(str) + str = strings.Replace(str, `\`, `\\`, -1) + str = strings.Replace(str, `%`, `\%`, -1) + str = strings.Replace(str, `_`, `\_`, -1) + return str } // ParamPlaceholderForIn returns a string that contains placeholders for sql keyword "in" diff --git a/src/lib/orm/orm_test.go b/src/lib/orm/test/orm_test.go similarity index 92% rename from src/lib/orm/orm_test.go rename to src/lib/orm/test/orm_test.go index 97745a5ed..d7a52c742 100644 --- a/src/lib/orm/orm_test.go +++ b/src/lib/orm/test/orm_test.go @@ -1,22 +1,23 @@ -// Copyright Project Harbor Authors +// Copyright Project Harbor Authors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package orm +package test import ( "context" "errors" + . "github.com/goharbor/harbor/src/lib/orm" "sync" "testing" diff --git a/src/pkg/authproxy/http.go b/src/pkg/authproxy/http.go index d9e76dcd9..56f6c64d1 100644 --- a/src/pkg/authproxy/http.go +++ b/src/pkg/authproxy/http.go @@ -6,6 +6,7 @@ import ( "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao/group" "github.com/goharbor/harbor/src/common/models" + cfgModels "github.com/goharbor/harbor/src/lib/config/models" "github.com/goharbor/harbor/src/lib/log" k8s_api_v1beta1 "k8s.io/api/authentication/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -16,7 +17,7 @@ import ( ) // TokenReview ... -func TokenReview(rawToken string, authProxyConfig *models.HTTPAuthProxy) (k8s_api_v1beta1.TokenReviewStatus, error) { +func TokenReview(rawToken string, authProxyConfig *cfgModels.HTTPAuthProxy) (k8s_api_v1beta1.TokenReviewStatus, error) { emptyStatus := k8s_api_v1beta1.TokenReviewStatus{} // Init auth client with the auth proxy endpoint. @@ -65,7 +66,7 @@ func TokenReview(rawToken string, authProxyConfig *models.HTTPAuthProxy) (k8s_ap } -func getTLSConfig(config *models.HTTPAuthProxy) rest.TLSClientConfig { +func getTLSConfig(config *cfgModels.HTTPAuthProxy) rest.TLSClientConfig { if config.VerifyCert && len(config.ServerCertificate) > 0 { return rest.TLSClientConfig{ CAData: []byte(config.ServerCertificate), diff --git a/src/pkg/authproxy/http_test.go b/src/pkg/authproxy/http_test.go index 2e65366a2..578bfd79f 100644 --- a/src/pkg/authproxy/http_test.go +++ b/src/pkg/authproxy/http_test.go @@ -3,7 +3,7 @@ package authproxy import ( "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/dao/group" - "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/lib/config/models" "github.com/stretchr/testify/assert" "k8s.io/api/authentication/v1beta1" "k8s.io/client-go/rest" diff --git a/src/common/config/store/driver/cache.go b/src/pkg/config/db/cache.go similarity index 50% rename from src/common/config/store/driver/cache.go rename to src/pkg/config/db/cache.go index 8117d9fbf..5f2bef082 100644 --- a/src/common/config/store/driver/cache.go +++ b/src/pkg/config/db/cache.go @@ -1,20 +1,22 @@ -// Copyright Project Harbor Authors +// Copyright Project Harbor Authors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package driver +package db import ( + "context" + "github.com/goharbor/harbor/src/pkg/config/store" "time" "github.com/goharbor/harbor/src/lib/cache" @@ -26,13 +28,13 @@ const cacheKey = "cfgs" // Cache - Used to load/save configuration with cache type Cache struct { cache cache.Cache - driver Driver + driver store.Driver } // Load - load config from database, only user setting will be load from database. -func (d *Cache) Load() (map[string]interface{}, error) { +func (d *Cache) Load(ctx context.Context) (map[string]interface{}, error) { f := func() (interface{}, error) { - return d.driver.Load() + return d.driver.Load(ctx) } result := map[string]interface{}{} @@ -47,8 +49,8 @@ func (d *Cache) Load() (map[string]interface{}, error) { } // Save - Only save user config items in the cfgs map -func (d *Cache) Save(cfgs map[string]interface{}) error { - if err := d.driver.Save(cfgs); err != nil { +func (d *Cache) Save(ctx context.Context, cfg map[string]interface{}) error { + if err := d.driver.Save(ctx, cfg); err != nil { return err } @@ -60,7 +62,7 @@ func (d *Cache) Save(cfgs map[string]interface{}) error { } // NewCacheDriver returns driver with cache -func NewCacheDriver(cache cache.Cache, driver Driver) Driver { +func NewCacheDriver(cache cache.Cache, driver store.Driver) store.Driver { return &Cache{ cache: cache, driver: driver, diff --git a/src/common/config/store/driver/cache_test.go b/src/pkg/config/db/cache_test.go similarity index 51% rename from src/common/config/store/driver/cache_test.go rename to src/pkg/config/db/cache_test.go index f8a079f8d..9065eae69 100644 --- a/src/common/config/store/driver/cache_test.go +++ b/src/pkg/config/db/cache_test.go @@ -1,41 +1,44 @@ -// Copyright Project Harbor Authors +// Copyright Project Harbor Authors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package driver +package db import ( "fmt" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/pkg/config/db/dao" "testing" "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/config/metadata" "github.com/goharbor/harbor/src/lib/cache" _ "github.com/goharbor/harbor/src/lib/cache/redis" // redis cache + "github.com/goharbor/harbor/src/lib/config/metadata" "github.com/stretchr/testify/assert" ) func TestCacheLoadAndSave(t *testing.T) { + ctx := orm.Context() cache, _ := cache.New("redis") - driver := NewCacheDriver(cache, &Database{}) + driver := NewCacheDriver(cache, &Database{cfgDAO: dao.New()}) cfgs := map[string]interface{}{ common.AUTHMode: "db_auth", common.LDAPURL: "ldap://ldap.vmware.com", } - driver.Save(cfgs) + driver.Save(orm.Context(), cfgs) - cf, err := driver.Load() + cf, err := driver.Load(ctx) if err != nil { fmt.Printf("load failed %v", err) } @@ -45,20 +48,21 @@ func TestCacheLoadAndSave(t *testing.T) { } func BenchmarkCacheLoad(b *testing.B) { + ctx := orm.Context() cfgs := map[string]interface{}{} for _, item := range metadata.Instance().GetAll() { cfgs[item.Name] = item.DefaultValue } driver := Database{} - driver.Save(cfgs) + driver.Save(ctx, cfgs) cache, _ := cache.New("redis") c := Cache{cache: cache, driver: &driver} b.RunParallel(func(pb *testing.PB) { for pb.Next() { - if _, err := c.Load(); err != nil { + if _, err := c.Load(ctx); err != nil { fmt.Printf("load failed, %v", err) } } diff --git a/src/pkg/config/db/dao/dao.go b/src/pkg/config/db/dao/dao.go new file mode 100644 index 000000000..6cacbcd06 --- /dev/null +++ b/src/pkg/config/db/dao/dao.go @@ -0,0 +1,106 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "context" + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/lib/config/models" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/orm" +) + +// DAO the dao for configure items +type DAO interface { + // GetConfigEntries get all configure items + GetConfigEntries(ctx context.Context) ([]*models.ConfigEntry, error) + // SaveConfigEntries save configure items provided + SaveConfigEntries(ctx context.Context, entries []models.ConfigEntry) error +} + +type dao struct { +} + +// New ... +func New() DAO { + return &dao{} +} + +func (d dao) GetConfigEntries(ctx context.Context) ([]*models.ConfigEntry, error) { + o, err := orm.FromContext(ctx) + if err != nil { + return nil, err + } + var p []*models.ConfigEntry + sql := "select * from properties" + n, err := o.Raw(sql, []interface{}{}).QueryRows(&p) + + if err != nil { + return nil, err + } + + if n == 0 { + return nil, nil + } + return p, nil +} + +func (d dao) SaveConfigEntries(ctx context.Context, entries []models.ConfigEntry) error { + o, err := orm.FromContext(ctx) + if err != nil { + return err + } + for _, entry := range entries { + if entry.Key == common.LDAPGroupAdminDn { + entry.Value = utils.TrimLower(entry.Value) + } + tempEntry := models.ConfigEntry{} + tempEntry.Key = entry.Key + tempEntry.Value = entry.Value + created, _, err := o.ReadOrCreate(&tempEntry, "k") + if err != nil && !orm.IsDuplicateKeyError(err) { + return errors.Wrap(err, "failed to create configuration entry") + } + if !created { + entry.ID = tempEntry.ID + _, err := o.Update(&entry, "v") + if err != nil { + return err + } + } + } + return nil +} + +// GetConfigEntries Get configuration from database +func GetConfigEntries(ctx context.Context) ([]*models.ConfigEntry, error) { + o, err := orm.FromContext(ctx) + if err != nil { + return nil, err + } + var p []*models.ConfigEntry + sql := "select * from properties" + n, err := o.Raw(sql, []interface{}{}).QueryRows(&p) + + if err != nil { + return nil, err + } + + if n == 0 { + return nil, nil + } + return p, nil +} diff --git a/src/pkg/config/db/dao/dao_test.go b/src/pkg/config/db/dao/dao_test.go new file mode 100644 index 000000000..b08580c21 --- /dev/null +++ b/src/pkg/config/db/dao/dao_test.go @@ -0,0 +1,132 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "context" + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/lib/config/models" + "github.com/goharbor/harbor/src/lib/orm" + "os" + "testing" +) + +var testCtx context.Context + +func TestMain(m *testing.M) { + test.InitDatabaseFromEnv() + testCtx = orm.Context() + os.Exit(m.Run()) +} + +func TestSaveConfigEntries(t *testing.T) { + dao := New() + configEntries := []models.ConfigEntry{ + { + Key: "teststringkey", + Value: "192.168.111.211", + }, + { + Key: "testboolkey", + Value: "true", + }, + { + Key: "testnumberkey", + Value: "5", + }, + { + Key: common.CfgDriverDB, + Value: "db", + }, + } + err := dao.SaveConfigEntries(testCtx, configEntries) + if err != nil { + t.Fatalf("failed to save configuration to database %v", err) + } + readEntries, err := GetConfigEntries(testCtx) + if err != nil { + t.Fatalf("Failed to get configuration from database %v", err) + } + findItem := 0 + for _, entry := range readEntries { + switch entry.Key { + case "teststringkey": + if "192.168.111.211" == entry.Value { + findItem++ + } + case "testnumberkey": + if "5" == entry.Value { + findItem++ + } + case "testboolkey": + if "true" == entry.Value { + findItem++ + } + default: + } + } + if findItem != 3 { + t.Fatalf("Should update 3 configuration but only update %d", findItem) + } + + configEntries = []models.ConfigEntry{ + { + Key: "teststringkey", + Value: "192.168.111.215", + }, + { + Key: "testboolkey", + Value: "false", + }, + { + Key: "testnumberkey", + Value: "7", + }, + { + Key: common.CfgDriverDB, + Value: "db", + }, + } + err = dao.SaveConfigEntries(testCtx, configEntries) + if err != nil { + t.Fatalf("failed to save configuration to database %v", err) + } + readEntries, err = GetConfigEntries(testCtx) + if err != nil { + t.Fatalf("Failed to get configuration from database %v", err) + } + findItem = 0 + for _, entry := range readEntries { + switch entry.Key { + case "teststringkey": + if "192.168.111.215" == entry.Value { + findItem++ + } + case "testnumberkey": + if "7" == entry.Value { + findItem++ + } + case "testboolkey": + if "false" == entry.Value { + findItem++ + } + default: + } + } + if findItem != 3 { + t.Fatalf("Should update 3 configuration but only update %d", findItem) + } +} diff --git a/src/common/config/store/driver/db.go b/src/pkg/config/db/db.go similarity index 62% rename from src/common/config/store/driver/db.go rename to src/pkg/config/db/db.go index d84f2a596..a2a696887 100644 --- a/src/common/config/store/driver/db.go +++ b/src/pkg/config/db/db.go @@ -1,38 +1,40 @@ -// Copyright Project Harbor Authors +// Copyright Project Harbor Authors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package driver +package db import ( + "context" + "github.com/goharbor/harbor/src/lib/config/models" + "github.com/goharbor/harbor/src/pkg/config/db/dao" + "github.com/goharbor/harbor/src/pkg/encrypt" "os" - "github.com/goharbor/harbor/src/common/config/encrypt" - "github.com/goharbor/harbor/src/common/config/metadata" - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/lib/config/metadata" "github.com/goharbor/harbor/src/lib/log" ) // Database - Used to load/save configuration in database type Database struct { + cfgDAO dao.DAO } // Load - load config from database, only user setting will be load from database. -func (d *Database) Load() (map[string]interface{}, error) { +func (d *Database) Load(ctx context.Context) (map[string]interface{}, error) { resultMap := map[string]interface{}{} - configEntries, err := dao.GetConfigEntries() + configEntries, err := d.cfgDAO.GetConfigEntries(ctx) if err != nil { return resultMap, err } @@ -59,7 +61,7 @@ func (d *Database) Load() (map[string]interface{}, error) { } // Save - Only save user config items in the cfgs map -func (d *Database) Save(cfgs map[string]interface{}) error { +func (d *Database) Save(ctx context.Context, cfgs map[string]interface{}) error { var configEntries []models.ConfigEntry for key, value := range cfgs { if item, ok := metadata.Instance().GetByName(key); ok { @@ -81,5 +83,5 @@ func (d *Database) Save(cfgs map[string]interface{}) error { log.Errorf("failed to get metadata, skip to save key:%v", key) } } - return dao.SaveConfigEntries(configEntries) + return d.cfgDAO.SaveConfigEntries(ctx, configEntries) } diff --git a/src/pkg/config/db/db_test.go b/src/pkg/config/db/db_test.go new file mode 100644 index 000000000..a73a0a45a --- /dev/null +++ b/src/pkg/config/db/db_test.go @@ -0,0 +1,79 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package db + +import ( + "fmt" + "testing" + + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/lib/config/metadata" + "github.com/goharbor/harbor/src/pkg/config/db/dao" + "github.com/stretchr/testify/assert" +) + +func TestDatabase_Load(t *testing.T) { + + cfgs := map[string]interface{}{ + common.AUTHMode: "db_auth", + common.LDAPURL: "ldap://ldap.vmware.com", + } + driver := Database{cfgDAO: dao.New()} + driver.Save(testCtx, cfgs) + cfgMap, err := driver.Load(testCtx) + if err != nil { + t.Errorf("failed to load, error %v", err) + } + assert.True(t, len(cfgMap) >= 1) + + if _, ok := cfgMap["ldap_url"]; !ok { + t.Error("Can not find ldap_url") + } +} + +func TestDatabase_Save(t *testing.T) { + ldapURL := "ldap://ldap.vmware.com" + driver := Database{cfgDAO: dao.New()} + prevCfg, err := driver.Load(testCtx) + if err != nil { + t.Errorf("failed to load config %v", err) + } + cfgMap := map[string]interface{}{"ldap_url": ldapURL} + driver.Save(testCtx, cfgMap) + updatedMap, err := driver.Load(testCtx) + if err != nil { + t.Errorf("failed to load config %v", err) + } + assert.Equal(t, updatedMap["ldap_url"], ldapURL) + driver.Save(testCtx, prevCfg) + +} + +func BenchmarkDatabaseLoad(b *testing.B) { + cfgs := map[string]interface{}{} + for _, item := range metadata.Instance().GetAll() { + cfgs[item.Name] = item.DefaultValue + } + + driver := Database{cfgDAO: dao.New()} + driver.Save(testCtx, cfgs) + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := driver.Load(testCtx); err != nil { + fmt.Printf("load failed %v", err) + } + } + }) +} diff --git a/src/pkg/config/db/manager.go b/src/pkg/config/db/manager.go new file mode 100644 index 000000000..48df44c25 --- /dev/null +++ b/src/pkg/config/db/manager.go @@ -0,0 +1,46 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package db + +import ( + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/lib/cache" + libCfg "github.com/goharbor/harbor/src/lib/config" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/config" + "github.com/goharbor/harbor/src/pkg/config/db/dao" + "github.com/goharbor/harbor/src/pkg/config/store" +) + +func init() { + libCfg.Register(common.DBCfgManager, NewDBCfgManager()) +} + +// NewDBCfgManager - create DB config manager +func NewDBCfgManager() *config.CfgManager { + cfgDriver := (store.Driver)(&Database{cfgDAO: dao.New()}) + + if cache.Default() != nil { + log.Debug("create DB config manager with cache enabled") + cfgDriver = NewCacheDriver(cache.Default(), cfgDriver) + } + + manager := &config.CfgManager{Store: store.NewConfigStore(cfgDriver)} + // load default value + manager.LoadDefault() + // load system config from env + manager.LoadSystemConfigFromEnv() + return manager +} diff --git a/src/pkg/config/db/manager_test.go b/src/pkg/config/db/manager_test.go new file mode 100644 index 000000000..00517b8eb --- /dev/null +++ b/src/pkg/config/db/manager_test.go @@ -0,0 +1,194 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package db + +import ( + "context" + "fmt" + "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/lib/config/metadata" + "github.com/goharbor/harbor/src/lib/orm" + cfgPkg "github.com/goharbor/harbor/src/pkg/config" + "github.com/goharbor/harbor/src/pkg/config/db/dao" + "github.com/goharbor/harbor/src/pkg/config/store" + "github.com/stretchr/testify/assert" + "os" + "testing" +) + +var TestDBConfig = map[string]interface{}{ + "postgresql_host": "localhost", + "postgresql_database": "registry", + "postgresql_password": "root123", + "postgresql_username": "postgres", + "postgresql_sslmode": "disable", + "email_host": "127.0.0.1", + "scan_all_policy": `{"parameter":{"daily_time":0},"type":"daily"}`, +} + +var configManager *cfgPkg.CfgManager +var testCtx context.Context + +func TestMain(m *testing.M) { + configManager = NewDBCfgManager() + test.InitDatabaseFromEnv() + testCtx = orm.Context() + configManager.UpdateConfig(testCtx, TestDBConfig) + os.Exit(m.Run()) +} + +func TestLoadFromDatabase(t *testing.T) { + configManager.UpdateConfig(testCtx, TestDBConfig) + configManager.Load(testCtx) + assert.Equal(t, "127.0.0.1", configManager.Get(testCtx, "email_host").GetString()) + assert.Equal(t, `{"parameter":{"daily_time":0},"type":"daily"}`, configManager.Get(testCtx, "scan_all_policy").GetString()) +} + +func TestLoadUserCfg(t *testing.T) { + configMap := configManager.GetUserCfgs(testCtx) + assert.NotNil(t, configMap["ldap_url"]) + assert.NotNil(t, configMap["ldap_base_dn"]) +} + +func TestSaveToDatabase(t *testing.T) { + fmt.Printf("database config %#v\n", configManager.GetDatabaseCfg()) + configManager.Load(testCtx) + configManager.Set(testCtx, "read_only", "true") + configManager.Save(testCtx) + configManager.Load(testCtx) + assert.Equal(t, true, configManager.Get(testCtx, "read_only").GetBool()) +} + +func TestUpdateCfg(t *testing.T) { + testConfig := map[string]interface{}{ + "ldap_url": "ldaps://ldap.vmware.com", + "ldap_search_dn": "cn=admin,dc=example,dc=com", + "ldap_timeout": 10, + "ldap_search_password": "admin", + "ldap_base_dn": "dc=example,dc=com", + } + configManager.Load(testCtx) + configManager.UpdateConfig(testCtx, testConfig) + + assert.Equal(t, "ldaps://ldap.vmware.com", configManager.Get(testCtx, "ldap_url").GetString()) + assert.Equal(t, 10, configManager.Get(testCtx, "ldap_timeout").GetInt()) + assert.Equal(t, "admin", configManager.Get(testCtx, "ldap_search_password").GetPassword()) + assert.Equal(t, "cn=admin,dc=example,dc=com", configManager.Get(testCtx, "ldap_search_dn").GetString()) + assert.Equal(t, "dc=example,dc=com", configManager.Get(testCtx, "ldap_base_dn").GetString()) +} + +func TestCfgManager_loadDefaultValues(t *testing.T) { + configManager.LoadDefault() + if configManager.Get(testCtx, "ldap_timeout").GetInt() != 5 { + t.Errorf("Failed to load ldap_timeout") + } +} + +func TestCfgManger_loadSystemValues(t *testing.T) { + configManager.LoadDefault() + configManager.LoadSystemConfigFromEnv() + configManager.UpdateConfig(testCtx, map[string]interface{}{ + "postgresql_host": "127.0.0.1", + }) + if configManager.Get(testCtx, "postgresql_host").GetString() != "127.0.0.1" { + t.Errorf("Failed to set system value postgresql_host, expected %v, actual %v", "127.0.0.1", configManager.Get(nil, "postgresql_host").GetString()) + } +} +func TestCfgManager_GetDatabaseCfg(t *testing.T) { + configManager.UpdateConfig(testCtx, map[string]interface{}{ + "postgresql_host": "localhost", + "postgresql_database": "registry", + "postgresql_password": "root123", + "postgresql_username": "postgres", + "postgresql_sslmode": "disable", + }) + dbCfg := configManager.GetDatabaseCfg() + assert.Equal(t, "localhost", dbCfg.PostGreSQL.Host) + assert.Equal(t, "registry", dbCfg.PostGreSQL.Database) + assert.Equal(t, "root123", dbCfg.PostGreSQL.Password) + assert.Equal(t, "postgres", dbCfg.PostGreSQL.Username) + assert.Equal(t, "disable", dbCfg.PostGreSQL.SSLMode) +} + +func TestConfigStore_Save(t *testing.T) { + cfgStore := store.NewConfigStore(&Database{cfgDAO: dao.New()}) + err := cfgStore.Save(testCtx) + cfgStore.Set("ldap_verify_cert", metadata.ConfigureValue{Name: "ldap_verify_cert", Value: "true"}) + if err != nil { + t.Fatal(err) + } + cfgValue, err := cfgStore.Get("ldap_verify_cert") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, true, cfgValue.GetBool()) + +} + +func TestConfigStore_Load(t *testing.T) { + cfgStore := store.NewConfigStore(&Database{cfgDAO: dao.New()}) + err := cfgStore.Load(testCtx) + if err != nil { + t.Fatal(err) + } + cfgValue, err := cfgStore.Get("ldap_url") + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, "ldaps://ldap.vmware.com", cfgValue.GetString()) + +} + +func TestToString(t *testing.T) { + cases := []struct { + name string + value interface{} + expect string + }{ + { + name: "transform int", + value: 999, + expect: "999", + }, + { + name: "transform slice", + value: []int{0, 1, 2}, + expect: "[0,1,2]", + }, + { + name: "transform map", + value: map[string]string{"k": "v"}, + expect: "{\"k\":\"v\"}", + }, + { + name: "transform bool", + value: false, + expect: "false", + }, + { + name: "transform nil", + value: nil, + expect: "nil", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + s, err := store.ToString(c.value) + assert.Nil(t, err) + assert.Equal(t, c.expect, s) + }) + } +} diff --git a/src/pkg/config/inmemory/manager.go b/src/pkg/config/inmemory/manager.go new file mode 100644 index 000000000..8fcf17a1b --- /dev/null +++ b/src/pkg/config/inmemory/manager.go @@ -0,0 +1,67 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inmemory + +import ( + "context" + "github.com/goharbor/harbor/src/common" + libCfg "github.com/goharbor/harbor/src/lib/config" + "github.com/goharbor/harbor/src/pkg/config" + "github.com/goharbor/harbor/src/pkg/config/store" + "sync" +) + +func init() { + libCfg.Register(common.InMemoryCfgManager, NewInMemoryManager()) +} + +// Driver driver for unit testing +type Driver struct { + sync.Mutex + cfgMap map[string]interface{} +} + +// Load load data from driver, for example load from database, +// it should be invoked before get any user scope config +// for system scope config, because it is immutable, no need to call this method +func (d *Driver) Load(context.Context) (map[string]interface{}, error) { + d.Lock() + defer d.Unlock() + res := make(map[string]interface{}) + for k, v := range d.cfgMap { + res[k] = v + } + return res, nil +} + +// Save only save user config setting to driver, for example: database, REST +func (d *Driver) Save(ctx context.Context, cfg map[string]interface{}) error { + d.Lock() + defer d.Unlock() + for k, v := range cfg { + d.cfgMap[k] = v + } + return nil +} + +// NewInMemoryManager create a manager for unit testing, doesn't involve database or REST +func NewInMemoryManager() *config.CfgManager { + manager := &config.CfgManager{Store: store.NewConfigStore(&Driver{cfgMap: map[string]interface{}{}})} + // load default Value + manager.LoadDefault() + // load system config from env + manager.LoadSystemConfigFromEnv() + return manager +} diff --git a/src/pkg/config/inmemory/manager_test.go b/src/pkg/config/inmemory/manager_test.go new file mode 100644 index 000000000..055bdda33 --- /dev/null +++ b/src/pkg/config/inmemory/manager_test.go @@ -0,0 +1,34 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inmemory + +import ( + "context" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNewInMemoryManager(t *testing.T) { + ctx := context.Background() + inMemoryManager := NewInMemoryManager() + inMemoryManager.UpdateConfig(ctx, map[string]interface{}{ + "ldap_url": "ldaps://ldap.vmware.com", + "ldap_timeout": 5, + "ldap_verify_cert": true, + }) + assert.Equal(t, "ldaps://ldap.vmware.com", inMemoryManager.Get(ctx, "ldap_url").GetString()) + assert.Equal(t, 5, inMemoryManager.Get(ctx, "ldap_timeout").GetInt()) + assert.Equal(t, true, inMemoryManager.Get(ctx, "ldap_verify_cert").GetBool()) +} diff --git a/src/pkg/config/manager.go b/src/pkg/config/manager.go new file mode 100644 index 000000000..f8602ee97 --- /dev/null +++ b/src/pkg/config/manager.go @@ -0,0 +1,186 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "context" + "fmt" + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils" + libCfg "github.com/goharbor/harbor/src/lib/config" + "github.com/goharbor/harbor/src/lib/config/metadata" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/config/store" + "os" +) + +// Manager ... +type Manager libCfg.Manager + +// CfgManager ... Configure Manager +type CfgManager struct { + Store *store.ConfigStore +} + +// LoadDefault ... +func (c *CfgManager) LoadDefault() { + // Init Default Value + itemArray := metadata.Instance().GetAll() + for _, item := range itemArray { + if len(item.DefaultValue) > 0 { + cfgValue, err := metadata.NewCfgValue(item.Name, item.DefaultValue) + if err != nil { + log.Errorf("LoadDefault failed, config item, key: %v, err: %v", item.Name, err) + continue + } + c.Store.Set(item.Name, *cfgValue) + } + } +} + +// LoadSystemConfigFromEnv ... +func (c *CfgManager) LoadSystemConfigFromEnv() { + itemArray := metadata.Instance().GetAll() + // Init System Value + for _, item := range itemArray { + if item.Scope == metadata.SystemScope && len(item.EnvKey) > 0 { + if envValue, ok := os.LookupEnv(item.EnvKey); ok { + configValue, err := metadata.NewCfgValue(item.Name, envValue) + if err != nil { + log.Errorf("LoadSystemConfigFromEnv failed, config item, key: %v, err: %v", item.Name, err) + continue + } + c.Store.Set(item.Name, *configValue) + } + } + } +} + +// GetAll get all settings. +func (c *CfgManager) GetAll(ctx context.Context) map[string]interface{} { + resultMap := map[string]interface{}{} + if err := c.Store.Load(ctx); err != nil { + log.Errorf("AllConfigs failed, error %v", err) + return resultMap + } + metaDataList := metadata.Instance().GetAll() + for _, item := range metaDataList { + cfgValue, err := c.Store.GetAnyType(item.Name) + if err != nil { + if err != metadata.ErrValueNotSet { + log.Errorf("Failed to get Value of key %v, error %v", item.Name, err) + } + continue + } + resultMap[item.Name] = cfgValue + } + return resultMap +} + +// GetUserCfgs retrieve all user configs +func (c *CfgManager) GetUserCfgs(ctx context.Context) map[string]interface{} { + resultMap := map[string]interface{}{} + if err := c.Store.Load(ctx); err != nil { + log.Errorf("UserConfigs failed, error %v", err) + return resultMap + } + metaDataList := metadata.Instance().GetAll() + for _, item := range metaDataList { + if item.Scope == metadata.UserScope { + cfgValue, err := c.Store.GetAnyType(item.Name) + if err != nil { + if err == metadata.ErrValueNotSet { + if _, ok := item.ItemType.(*metadata.StringType); ok { + cfgValue = "" + } + if _, ok := item.ItemType.(*metadata.NonEmptyStringType); ok { + cfgValue = "" + } + } else { + log.Errorf("Failed to get Value of key %v, error %v", item.Name, err) + continue + } + } + resultMap[item.Name] = cfgValue + } + } + return resultMap +} + +// Load load configuration from storage, like database or redis +func (c *CfgManager) Load(ctx context.Context) error { + return c.Store.Load(ctx) +} + +// Save - Save all current configuration to storage +func (c *CfgManager) Save(ctx context.Context) error { + return c.Store.Save(ctx) +} + +// Get ... +func (c *CfgManager) Get(ctx context.Context, key string) *metadata.ConfigureValue { + configValue, err := c.Store.Get(key) + if err != nil { + log.Debugf("failed to get key %v, error: %v", key, err) + configValue = &metadata.ConfigureValue{} + } + return configValue +} + +// Set ... +func (c *CfgManager) Set(ctx context.Context, key string, value interface{}) { + configValue, err := metadata.NewCfgValue(key, utils.GetStrValueOfAnyType(value)) + if err != nil { + log.Errorf("error when setting key: %v, error %v", key, err) + return + } + c.Store.Set(key, *configValue) +} + +// GetDatabaseCfg - Get database configurations +func (c *CfgManager) GetDatabaseCfg() *models.Database { + ctx := context.Background() + return &models.Database{ + Type: c.Get(ctx, common.DatabaseType).GetString(), + PostGreSQL: &models.PostGreSQL{ + Host: c.Get(ctx, common.PostGreSQLHOST).GetString(), + Port: c.Get(ctx, common.PostGreSQLPort).GetInt(), + Username: c.Get(ctx, common.PostGreSQLUsername).GetString(), + Password: c.Get(ctx, common.PostGreSQLPassword).GetString(), + Database: c.Get(ctx, common.PostGreSQLDatabase).GetString(), + SSLMode: c.Get(ctx, common.PostGreSQLSSLMode).GetString(), + MaxIdleConns: c.Get(ctx, common.PostGreSQLMaxIdleConns).GetInt(), + MaxOpenConns: c.Get(ctx, common.PostGreSQLMaxOpenConns).GetInt(), + }, + } +} + +// UpdateConfig - Update config Store with a specified configuration and also save updated configure. +func (c *CfgManager) UpdateConfig(ctx context.Context, cfgs map[string]interface{}) error { + return c.Store.Update(ctx, cfgs) +} + +// ValidateCfg validate config by metadata. return the first error if exist. +func (c *CfgManager) ValidateCfg(ctx context.Context, cfgs map[string]interface{}) error { + for key, value := range cfgs { + strVal := utils.GetStrValueOfAnyType(value) + _, err := metadata.NewCfgValue(key, strVal) + if err != nil { + return fmt.Errorf("%v, item name: %v", err, key) + } + } + return nil +} diff --git a/src/pkg/config/rest/manager.go b/src/pkg/config/rest/manager.go new file mode 100644 index 000000000..36f7e3dc1 --- /dev/null +++ b/src/pkg/config/rest/manager.go @@ -0,0 +1,53 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rest + +import ( + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/http/modifier/auth" + libCfg "github.com/goharbor/harbor/src/lib/config" + "github.com/goharbor/harbor/src/pkg/config" + "github.com/goharbor/harbor/src/pkg/config/store" + "os" + "strings" +) + +func init() { + unitTest := os.Getenv("UTTEST") + if strings.EqualFold(unitTest, "true") == true { + libCfg.Register(common.RestCfgManager, NewRESTCfgManager("sample_url", "sample_secret")) + return + } + + coreURL := os.Getenv("CORE_URL") + if len(coreURL) == 0 { + return + } + configURL := coreURL + common.CoreConfigPath + jobSvcSecret := os.Getenv("JOBSERVICE_SECRET") + if len(jobSvcSecret) == 0 { + return + } + // by default rest config manager is used by jobservice + // for other scenario, should change the initialization of config manager + libCfg.Register(common.RestCfgManager, NewRESTCfgManager(configURL, jobSvcSecret)) +} + +// NewRESTCfgManager - create REST config manager +func NewRESTCfgManager(configURL, secret string) *config.CfgManager { + secAuth := auth.NewSecretAuthorizer(secret) + manager := &config.CfgManager{Store: store.NewConfigStore(NewRESTDriver(configURL, secAuth))} + return manager +} diff --git a/src/pkg/config/rest/rest.go b/src/pkg/config/rest/rest.go new file mode 100644 index 000000000..a1fe81708 --- /dev/null +++ b/src/pkg/config/rest/rest.go @@ -0,0 +1,70 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rest + +import ( + "context" + "errors" + "net/http" + + commonhttp "github.com/goharbor/harbor/src/common/http" + "github.com/goharbor/harbor/src/common/http/modifier" + "github.com/goharbor/harbor/src/lib/log" +) + +// Driver - config store driver based on REST API +type Driver struct { + configRESTURL string + client *commonhttp.Client +} + +// NewRESTDriver - Create Driver +func NewRESTDriver(configRESTURL string, modifiers ...modifier.Modifier) *Driver { + if commonhttp.InternalTLSEnabled() { + tr := commonhttp.GetHTTPTransport(commonhttp.SecureTransport) + return &Driver{configRESTURL: configRESTURL, client: commonhttp.NewClient(&http.Client{Transport: tr}, modifiers...)} + + } + return &Driver{configRESTURL: configRESTURL, client: commonhttp.NewClient(nil, modifiers...)} +} + +// Value ... +type Value struct { + Val interface{} `json:"value"` + Editable bool `json:"editable"` +} + +// Load - load config data from REST server +func (h *Driver) Load(ctx context.Context) (map[string]interface{}, error) { + cfgMap := map[string]interface{}{} + origMap := map[string]*Value{} + log.Infof("get configuration from url: %+v", h.configRESTURL) + err := h.client.Get(h.configRESTURL, &origMap) + if err != nil { + log.Errorf("Failed on load rest config err:%v, url:%v", err, h.configRESTURL) + } + if len(origMap) < 1 { + return cfgMap, errors.New("failed to load rest config") + } + for k, v := range origMap { + cfgMap[k] = v.Val + } + return cfgMap, err +} + +// Save - Save config data to REST server by PUT method +func (h *Driver) Save(ctx context.Context, cfg map[string]interface{}) error { + return h.client.Put(h.configRESTURL, cfg) +} diff --git a/src/common/config/store/driver/rest_test.go b/src/pkg/config/rest/rest_test.go similarity index 65% rename from src/common/config/store/driver/rest_test.go rename to src/pkg/config/rest/rest_test.go index e7b887730..3c4411b4d 100644 --- a/src/common/config/store/driver/rest_test.go +++ b/src/pkg/config/rest/rest_test.go @@ -1,6 +1,21 @@ -package driver +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rest import ( + "context" "encoding/json" "github.com/goharbor/harbor/src/lib/log" "github.com/stretchr/testify/assert" @@ -12,9 +27,9 @@ import ( func ConfigGetHandler(w http.ResponseWriter, r *http.Request) { cfgs := map[string]interface{}{ - "ldap_url": "ldaps://ldap.vmware.com", - "ldap_scope": 5, - "ldap_verify_cert": true, + "ldap_url": &Value{Val: "ldaps://ldap.vmware.com", Editable: true}, + "ldap_scope": &Value{Val: 5, Editable: true}, + "ldap_verify_cert": &Value{Val: true, Editable: true}, } b, err := json.Marshal(cfgs) if err != nil { @@ -27,7 +42,7 @@ func TestHTTPDriver_Load(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(ConfigGetHandler)) defer server.Close() httpDriver := NewRESTDriver(server.URL) - configMap, err := httpDriver.Load() + configMap, err := httpDriver.Load(context.Background()) if err != nil { t.Errorf("Error when testing http driver %v", err) } @@ -63,7 +78,7 @@ func TestHTTPDriver_Save(t *testing.T) { "ldap_timeout": 10, "ldap_verify_cert": false, } - err := httpDriver.Save(configMap) + err := httpDriver.Save(context.Background(), configMap) if err != nil { t.Fatal(err) } diff --git a/src/pkg/config/store/driver.go b/src/pkg/config/store/driver.go new file mode 100644 index 000000000..7076319d7 --- /dev/null +++ b/src/pkg/config/store/driver.go @@ -0,0 +1,26 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package store is only used in the internal implement of manager, not a public api. +package store + +import "context" + +// Driver the interface to save/load config +type Driver interface { + // Load - load config item from config driver + Load(ctx context.Context) (map[string]interface{}, error) + // Save - save config item into config driver + Save(ctx context.Context, cfg map[string]interface{}) error +} diff --git a/src/common/config/store/store.go b/src/pkg/config/store/store.go similarity index 86% rename from src/common/config/store/store.go rename to src/pkg/config/store/store.go index 1eae74bd0..5b8340c4d 100644 --- a/src/common/config/store/store.go +++ b/src/pkg/config/store/store.go @@ -2,12 +2,12 @@ package store import ( + "context" "encoding/json" "errors" "fmt" - "github.com/goharbor/harbor/src/common/config/metadata" - "github.com/goharbor/harbor/src/common/config/store/driver" "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/lib/config/metadata" "github.com/goharbor/harbor/src/lib/log" "reflect" "strconv" @@ -16,12 +16,12 @@ import ( // ConfigStore - the config data store type ConfigStore struct { - cfgDriver driver.Driver + cfgDriver Driver cfgValues sync.Map } // NewConfigStore create config store -func NewConfigStore(cfgDriver driver.Driver) *ConfigStore { +func NewConfigStore(cfgDriver Driver) *ConfigStore { return &ConfigStore{cfgDriver: cfgDriver} } @@ -54,16 +54,16 @@ func (c *ConfigStore) Set(key string, value metadata.ConfigureValue) error { } // Load - Load data from driver, all user config in the store will be refreshed -func (c *ConfigStore) Load() error { +func (c *ConfigStore) Load(ctx context.Context) error { if c.cfgDriver == nil { return errors.New("failed to load store, cfgDriver is nil") } - cfgs, err := c.cfgDriver.Load() + cfgs, err := c.cfgDriver.Load(ctx) if err != nil { return err } for key, value := range cfgs { - strValue, err := toString(value) + strValue, err := ToString(value) if err != nil { log.Errorf("failed to transform the value from driver to string, key: %s, value: %v, error: %v", key, value, err) continue @@ -80,7 +80,7 @@ func (c *ConfigStore) Load() error { } // Save - Save all data in current store -func (c *ConfigStore) Save() error { +func (c *ConfigStore) Save(ctx context.Context) error { cfgMap := map[string]interface{}{} c.cfgValues.Range(func(key, value interface{}) bool { keyStr := fmt.Sprintf("%v", key) @@ -99,11 +99,11 @@ func (c *ConfigStore) Save() error { return errors.New("failed to save store, cfgDriver is nil") } - return c.cfgDriver.Save(cfgMap) + return c.cfgDriver.Save(ctx, cfgMap) } // Update - Only update specified settings in cfgMap in store and driver -func (c *ConfigStore) Update(cfgMap map[string]interface{}) error { +func (c *ConfigStore) Update(ctx context.Context, cfgMap map[string]interface{}) error { // Update to store for key, value := range cfgMap { configValue, err := metadata.NewCfgValue(key, utils.GetStrValueOfAnyType(value)) @@ -115,10 +115,11 @@ func (c *ConfigStore) Update(cfgMap map[string]interface{}) error { c.Set(key, *configValue) } // Update to driver - return c.cfgDriver.Save(cfgMap) + return c.cfgDriver.Save(ctx, cfgMap) } -func toString(value interface{}) (string, error) { +// ToString ... +func ToString(value interface{}) (string, error) { if value == nil { return "nil", nil } diff --git a/src/common/config/encrypt/encrypt.go b/src/pkg/encrypt/encrypt.go similarity index 71% rename from src/common/config/encrypt/encrypt.go rename to src/pkg/encrypt/encrypt.go index 9a26c7f9c..b81ffab2b 100644 --- a/src/common/config/encrypt/encrypt.go +++ b/src/pkg/encrypt/encrypt.go @@ -1,16 +1,16 @@ -// Copyright Project Harbor Authors +// Copyright Project Harbor Authors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package encrypt diff --git a/src/common/config/encrypt/encrypt_test.go b/src/pkg/encrypt/encrypt_test.go similarity index 58% rename from src/common/config/encrypt/encrypt_test.go rename to src/pkg/encrypt/encrypt_test.go index 019f88d84..ef592458c 100644 --- a/src/common/config/encrypt/encrypt_test.go +++ b/src/pkg/encrypt/encrypt_test.go @@ -1,3 +1,17 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package encrypt import ( diff --git a/src/common/config/keyprovider.go b/src/pkg/encrypt/keyprovider.go similarity index 62% rename from src/common/config/keyprovider.go rename to src/pkg/encrypt/keyprovider.go index 163ced369..f0204486b 100644 --- a/src/common/config/keyprovider.go +++ b/src/pkg/encrypt/keyprovider.go @@ -1,18 +1,18 @@ -// Copyright Project Harbor Authors +// Copyright Project Harbor Authors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package config +package encrypt import ( "io/ioutil" diff --git a/src/pkg/encrypt/keyprovider_test.go b/src/pkg/encrypt/keyprovider_test.go new file mode 100644 index 000000000..2f9576fc2 --- /dev/null +++ b/src/pkg/encrypt/keyprovider_test.go @@ -0,0 +1,44 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package encrypt + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestGetOfFileKeyProvider(t *testing.T) { + path := "/tmp/key" + key := "key_content" + + if err := ioutil.WriteFile(path, []byte(key), 0777); err != nil { + t.Errorf("failed to write to file %s: %v", path, err) + return + } + defer os.Remove(path) + + provider := NewFileKeyProvider(path) + k, err := provider.Get(nil) + if err != nil { + t.Errorf("failed to get key from the file provider: %v", err) + return + } + + if k != key { + t.Errorf("unexpected key: %s != %s", k, key) + return + } +} diff --git a/src/pkg/ldap/ldap.go b/src/pkg/ldap/ldap.go index 7930b3f06..b5ce0b974 100644 --- a/src/pkg/ldap/ldap.go +++ b/src/pkg/ldap/ldap.go @@ -371,6 +371,8 @@ func (s *Session) searchGroup(groupDN, filter, gName, groupNameAttribute string) groupName := "" if len(result.Entries[0].Attributes) > 0 { groupName = result.Entries[0].Attributes[0].Values[0] + } else { + groupName = groupDN } group := model.Group{ Dn: result.Entries[0].DN, diff --git a/src/pkg/notification/hook/hook.go b/src/pkg/notification/hook/hook.go index 2181160be..06cd5fdb8 100755 --- a/src/pkg/notification/hook/hook.go +++ b/src/pkg/notification/hook/hook.go @@ -4,12 +4,12 @@ import ( "context" "encoding/json" "fmt" + "github.com/goharbor/harbor/src/controller/config" "time" cJob "github.com/goharbor/harbor/src/common/job" "github.com/goharbor/harbor/src/common/job/models" cModels "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/utils" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/notification/job" diff --git a/src/pkg/oidc/helper.go b/src/pkg/oidc/helper.go index 26f19449e..d2497482e 100644 --- a/src/pkg/oidc/helper.go +++ b/src/pkg/oidc/helper.go @@ -19,6 +19,9 @@ import ( "crypto/tls" "errors" "fmt" + "github.com/goharbor/harbor/src/controller/config" + cfgModels "github.com/goharbor/harbor/src/lib/config/models" + "github.com/goharbor/harbor/src/lib/orm" "net/http" "strings" "sync" @@ -29,7 +32,6 @@ import ( "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao/group" "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" "golang.org/x/oauth2" ) @@ -81,7 +83,7 @@ func (p *providerHelper) get() (*gooidc.Provider, error) { } func (p *providerHelper) reloadSetting() error { - conf, err := config.OIDCSetting() + conf, err := config.OIDCSetting(orm.Context()) if err != nil { return fmt.Errorf("failed to load OIDC setting: %v", err) } @@ -93,7 +95,7 @@ func (p *providerHelper) create() error { if p.setting.Load() == nil { return errors.New("the configuration is not loaded") } - s := p.setting.Load().(models.OIDCSetting) + s := p.setting.Load().(cfgModels.OIDCSetting) ctx := clientCtx(context.Background(), s.VerifyCert) provider, err := gooidc.NewProvider(ctx, s.Endpoint) if err != nil { @@ -136,7 +138,7 @@ func getOauthConf() (*oauth2.Config, error) { if err != nil { return nil, err } - setting := provider.setting.Load().(models.OIDCSetting) + setting := provider.setting.Load().(cfgModels.OIDCSetting) scopes := make([]string, 0) for _, sc := range setting.Scope { if strings.HasPrefix(p.Endpoint().AuthURL, googleEndpoint) && sc == gooidc.ScopeOfflineAccess { @@ -163,7 +165,7 @@ func AuthCodeURL(state string) (string, error) { return "", err } var options []oauth2.AuthCodeOption - setting := provider.setting.Load().(models.OIDCSetting) + setting := provider.setting.Load().(cfgModels.OIDCSetting) for k, v := range setting.ExtraRedirectParms { options = append(options, oauth2.SetAuthURLParam(k, v)) } @@ -181,7 +183,7 @@ func ExchangeToken(ctx context.Context, code string) (*Token, error) { log.Errorf("Failed to get OAuth configuration, error: %v", err) return nil, err } - setting := provider.setting.Load().(models.OIDCSetting) + setting := provider.setting.Load().(cfgModels.OIDCSetting) ctx = clientCtx(ctx, setting.VerifyCert) oauthToken, err := oauth.Exchange(ctx, code) if err != nil { @@ -206,7 +208,7 @@ func verifyTokenWithConfig(ctx context.Context, rawIDToken string, conf *gooidc. if err != nil { return nil, err } - settings := provider.setting.Load().(models.OIDCSetting) + settings := provider.setting.Load().(cfgModels.OIDCSetting) if conf == nil { conf = &gooidc.Config{ClientID: settings.ClientID} } @@ -234,7 +236,7 @@ func refreshToken(ctx context.Context, token *Token) (*Token, error) { if err != nil { return nil, err } - setting := provider.setting.Load().(models.OIDCSetting) + setting := provider.setting.Load().(cfgModels.OIDCSetting) cctx := clientCtx(ctx, setting.VerifyCert) ts := oauthCfg.TokenSource(cctx, &token.Token) nt, err := ts.Token() @@ -256,7 +258,7 @@ func UserInfoFromToken(ctx context.Context, token *Token) (*UserInfo, error) { if err != nil { return nil, err } - setting := provider.setting.Load().(models.OIDCSetting) + setting := provider.setting.Load().(cfgModels.OIDCSetting) local, err := UserInfoFromIDToken(ctx, token, setting) if err != nil { return nil, err @@ -302,7 +304,7 @@ func mergeUserInfo(remote, local *UserInfo) *UserInfo { return res } -func userInfoFromRemote(ctx context.Context, token *Token, setting models.OIDCSetting) (*UserInfo, error) { +func userInfoFromRemote(ctx context.Context, token *Token, setting cfgModels.OIDCSetting) (*UserInfo, error) { p, err := provider.get() if err != nil { return nil, err @@ -316,7 +318,7 @@ func userInfoFromRemote(ctx context.Context, token *Token, setting models.OIDCSe } // UserInfoFromIDToken extract user info from ID token -func UserInfoFromIDToken(ctx context.Context, token *Token, setting models.OIDCSetting) (*UserInfo, error) { +func UserInfoFromIDToken(ctx context.Context, token *Token, setting cfgModels.OIDCSetting) (*UserInfo, error) { if token.RawIDToken == "" { return nil, nil } @@ -328,7 +330,7 @@ func UserInfoFromIDToken(ctx context.Context, token *Token, setting models.OIDCS return userInfoFromClaims(idt, setting) } -func userInfoFromClaims(c claimsProvider, setting models.OIDCSetting) (*UserInfo, error) { +func userInfoFromClaims(c claimsProvider, setting cfgModels.OIDCSetting) (*UserInfo, error) { res := &UserInfo{} if err := c.Claims(res); err != nil { return nil, err diff --git a/src/pkg/oidc/helper_test.go b/src/pkg/oidc/helper_test.go index b83065ec3..19882f81f 100644 --- a/src/pkg/oidc/helper_test.go +++ b/src/pkg/oidc/helper_test.go @@ -16,6 +16,11 @@ package oidc import ( "encoding/json" + "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/controller/config" + cfgModels "github.com/goharbor/harbor/src/lib/config/models" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/pkg/encrypt" "net/url" "os" "strings" @@ -23,13 +28,12 @@ import ( "time" "github.com/goharbor/harbor/src/common" - config2 "github.com/goharbor/harbor/src/common/config" "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/core/config" "github.com/stretchr/testify/assert" ) func TestMain(m *testing.M) { + test.InitDatabaseFromEnv() conf := map[string]interface{}{ common.OIDCName: "test", common.OIDCEndpoint: "https://accounts.google.com", @@ -39,7 +43,7 @@ func TestMain(m *testing.M) { common.OIDCClientSecret: "secret", common.ExtEndpoint: "https://harbor.test", } - kp := &config2.PresetKeyProvider{Key: "naa4JtarA1Zsc3uY"} + kp := &encrypt.PresetKeyProvider{Key: "naa4JtarA1Zsc3uY"} config.InitWithSettings(conf, kp) @@ -53,7 +57,7 @@ func TestHelperLoadConf(t *testing.T) { assert.Nil(t, testP.setting.Load()) err := testP.reloadSetting() assert.Nil(t, err) - assert.Equal(t, "test", testP.setting.Load().(models.OIDCSetting).Name) + assert.Equal(t, "test", testP.setting.Load().(cfgModels.OIDCSetting).Name) } func TestHelperCreate(t *testing.T) { @@ -82,11 +86,12 @@ func TestHelperGet(t *testing.T) { common.OIDCClientSecret: "new-secret", common.ExtEndpoint: "https://harbor.test", } - config.GetCfgManager().UpdateConfig(update) + ctx := orm.Context() + config.GetCfgManager(ctx).UpdateConfig(ctx, update) t.Log("Sleep for 5 seconds") time.Sleep(5 * time.Second) - assert.Equal(t, "new-secret", testP.setting.Load().(models.OIDCSetting).ClientSecret) + assert.Equal(t, "new-secret", testP.setting.Load().(cfgModels.OIDCSetting).ClientSecret) } func TestAuthCodeURL(t *testing.T) { @@ -100,7 +105,8 @@ func TestAuthCodeURL(t *testing.T) { common.ExtEndpoint: "https://harbor.test", common.OIDCExtraRedirectParms: `{"test_key":"test_value"}`, } - config.GetCfgManager().UpdateConfig(conf) + ctx := orm.Context() + config.GetCfgManager(ctx).UpdateConfig(ctx, conf) res, err := AuthCodeURL("random") assert.Nil(t, err) u, err := url.ParseRequestURI(res) @@ -187,7 +193,7 @@ func TestGroupsFromClaim(t *testing.T) { func TestUserInfoFromClaims(t *testing.T) { s := []struct { input map[string]interface{} - setting models.OIDCSetting + setting cfgModels.OIDCSetting expect *UserInfo }{ { @@ -196,7 +202,7 @@ func TestUserInfoFromClaims(t *testing.T) { "email": "daniel@gmail.com", "groups": []interface{}{"g1", "g2"}, }, - setting: models.OIDCSetting{ + setting: cfgModels.OIDCSetting{ Name: "t1", GroupsClaim: "grouplist", UserClaim: "", @@ -217,7 +223,7 @@ func TestUserInfoFromClaims(t *testing.T) { "email": "daniel@gmail.com", "groups": []interface{}{"g1", "g2"}, }, - setting: models.OIDCSetting{ + setting: cfgModels.OIDCSetting{ Name: "t2", GroupsClaim: "groups", UserClaim: "", @@ -241,7 +247,7 @@ func TestUserInfoFromClaims(t *testing.T) { "email": "jack@gmail.com", "groupclaim": []interface{}{}, }, - setting: models.OIDCSetting{ + setting: cfgModels.OIDCSetting{ Name: "t3", GroupsClaim: "groupclaim", UserClaim: "", @@ -263,7 +269,7 @@ func TestUserInfoFromClaims(t *testing.T) { "email": "airadier@gmail.com", "groups": []interface{}{"g1", "g2"}, }, - setting: models.OIDCSetting{ + setting: cfgModels.OIDCSetting{ Name: "t4", GroupsClaim: "grouplist", UserClaim: "email", diff --git a/src/pkg/oidc/secret.go b/src/pkg/oidc/secret.go index bc7b144d4..98126bd0b 100644 --- a/src/pkg/oidc/secret.go +++ b/src/pkg/oidc/secret.go @@ -4,12 +4,12 @@ import ( "context" "encoding/json" "fmt" + "github.com/goharbor/harbor/src/controller/config" "sync" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" ) diff --git a/src/pkg/quota/dao/util.go b/src/pkg/quota/dao/util.go index 129660367..54f7f8ad1 100644 --- a/src/pkg/quota/dao/util.go +++ b/src/pkg/quota/dao/util.go @@ -17,10 +17,10 @@ package dao import ( "encoding/json" "fmt" + "github.com/goharbor/harbor/src/lib/orm" "github.com/lib/pq" "strings" - "github.com/goharbor/harbor/src/lib/orm" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/quota/types" ) diff --git a/src/pkg/reg/manager.go b/src/pkg/reg/manager.go index bb2323f10..14ab2d831 100644 --- a/src/pkg/reg/manager.go +++ b/src/pkg/reg/manager.go @@ -17,6 +17,7 @@ package reg import ( "context" commonthttp "github.com/goharbor/harbor/src/common/http" + "github.com/goharbor/harbor/src/controller/config" // register the Harbor adapter _ "github.com/goharbor/harbor/src/pkg/reg/adapter/harbor" @@ -52,7 +53,6 @@ import ( _ "github.com/goharbor/harbor/src/pkg/reg/adapter/githubcr" "github.com/goharbor/harbor/src/common/utils" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/reg/adapter" "github.com/goharbor/harbor/src/pkg/reg/dao" diff --git a/src/pkg/registry/client.go b/src/pkg/registry/client.go index ee053c814..dbbf7332d 100644 --- a/src/pkg/registry/client.go +++ b/src/pkg/registry/client.go @@ -18,6 +18,7 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/goharbor/harbor/src/controller/config" "io" "io/ioutil" "net/http" @@ -32,7 +33,6 @@ import ( "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" commonhttp "github.com/goharbor/harbor/src/common/http" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/pkg/registry/auth" diff --git a/src/pkg/scan/job.go b/src/pkg/scan/job.go index b17ff49e8..90d116738 100644 --- a/src/pkg/scan/job.go +++ b/src/pkg/scan/job.go @@ -27,11 +27,11 @@ import ( "time" "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/config" commonhttp "github.com/goharbor/harbor/src/common/http" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/lib/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/pkg/robot/model" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" @@ -455,7 +455,7 @@ func getInternalTokenServiceEndpoint(ctx job.Context) (string, error) { return "", errors.Errorf("failed to get config manager") } - return cfgMgr.Get(common.CoreURL).GetString() + "/service/token", nil + return cfgMgr.Get(ctx.SystemContext(), common.CoreURL).GetString() + "/service/token", nil } // makeBasicAuthorization creates authorization from a robot account based on the arguments for scanning. diff --git a/src/pkg/signature/manager.go b/src/pkg/signature/manager.go index 0dee140db..df9664f6a 100644 --- a/src/pkg/signature/manager.go +++ b/src/pkg/signature/manager.go @@ -16,7 +16,7 @@ package signature import ( "github.com/goharbor/harbor/src/common/security" - "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/signature/notary" "github.com/goharbor/harbor/src/pkg/signature/notary/model" @@ -83,7 +83,7 @@ func (m *mgr) getTargetsByRepo(ctx context.Context, repo string) ([]model.Target } else { name = sc.GetUsername() } - return notary.GetInternalTargets(config.InternalNotaryEndpoint(), name, repo) + return notary.GetInternalTargets(ctx, config.InternalNotaryEndpoint(), name, repo) } var instance = &mgr{} diff --git a/src/pkg/signature/manager_test.go b/src/pkg/signature/manager_test.go index d15e9345f..6f3ecc44a 100644 --- a/src/pkg/signature/manager_test.go +++ b/src/pkg/signature/manager_test.go @@ -2,7 +2,8 @@ package signature import ( "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/core/config" + testutils "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/pkg/signature/notary/test" "github.com/stretchr/testify/assert" "golang.org/x/net/context" @@ -11,6 +12,7 @@ import ( ) func TestMain(m *testing.M) { + testutils.InitDatabaseFromEnv() // B/C the notary requires private key for signing token, b // before running locally, please make sure the env var is set as follow: // export TOKEN_PRIVATE_KEY_PATH="/harbor/tests/private_key.pem" diff --git a/src/pkg/signature/notary/helper.go b/src/pkg/signature/notary/helper.go index 96f8022e3..eac7c5b03 100644 --- a/src/pkg/signature/notary/helper.go +++ b/src/pkg/signature/notary/helper.go @@ -15,8 +15,10 @@ package notary import ( + "context" "encoding/hex" "fmt" + "github.com/goharbor/harbor/src/controller/config" "net/http" "os" "path" @@ -24,7 +26,6 @@ import ( "github.com/docker/distribution/registry/auth/token" commonhttp "github.com/goharbor/harbor/src/common/http" - "github.com/goharbor/harbor/src/core/config" tokenutil "github.com/goharbor/harbor/src/core/service/token" "github.com/goharbor/harbor/src/lib/log" model2 "github.com/goharbor/harbor/src/pkg/signature/notary/model" @@ -53,7 +54,7 @@ func init() { } // GetInternalTargets wraps GetTargets to read config values for getting full-qualified repo from internal notary instance. -func GetInternalTargets(notaryEndpoint string, username string, repo string) ([]model2.Target, error) { +func GetInternalTargets(ctx context.Context, notaryEndpoint string, username string, repo string) ([]model2.Target, error) { ext, err := config.ExtEndpoint() if err != nil { log.Errorf("Error while reading external endpoint: %v", err) @@ -61,15 +62,15 @@ func GetInternalTargets(notaryEndpoint string, username string, repo string) ([] } endpoint := strings.Split(ext, "//")[1] fqRepo := path.Join(endpoint, repo) - return GetTargets(notaryEndpoint, username, fqRepo) + return GetTargets(ctx, notaryEndpoint, username, fqRepo) } // GetTargets is a help function called by API to fetch signature information of a given repository. // Per docker's convention the repository should contain the information of endpoint, i.e. it should look // like "192.168.0.1/library/ubuntu", instead of "library/ubuntu" (fqRepo for fully-qualified repo) -func GetTargets(notaryEndpoint string, username string, fqRepo string) ([]model2.Target, error) { +func GetTargets(ctx context.Context, notaryEndpoint string, username string, fqRepo string) ([]model2.Target, error) { res := []model2.Target{} - t, err := tokenutil.MakeToken(username, tokenutil.Notary, + t, err := tokenutil.MakeToken(ctx, username, tokenutil.Notary, []*token.ResourceActions{ { Type: "repository", diff --git a/src/pkg/signature/notary/helper_test.go b/src/pkg/signature/notary/helper_test.go index b7b8175a5..53ca70d0a 100644 --- a/src/pkg/signature/notary/helper_test.go +++ b/src/pkg/signature/notary/helper_test.go @@ -16,11 +16,12 @@ package notary import ( "encoding/json" "fmt" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" model2 "github.com/goharbor/harbor/src/pkg/signature/notary/model" test2 "github.com/goharbor/harbor/src/pkg/signature/notary/test" "github.com/goharbor/harbor/src/common/utils/test" - "github.com/goharbor/harbor/src/core/config" "github.com/stretchr/testify/assert" "net/http/httptest" @@ -54,19 +55,19 @@ func TestMain(m *testing.M) { } func TestGetInternalTargets(t *testing.T) { - targets, err := GetInternalTargets(notaryServer.URL, "admin", "library/busybox") + targets, err := GetInternalTargets(orm.Context(), notaryServer.URL, "admin", "library/busybox") assert.Nil(t, err, fmt.Sprintf("Unexpected error: %v", err)) assert.Equal(t, 1, len(targets), "") assert.Equal(t, "latest-signed", targets[0].Tag, "") } func TestGetTargets(t *testing.T) { - targets, err := GetTargets(notaryServer.URL, "admin", path.Join(endpoint, "library/busybox")) + targets, err := GetTargets(orm.Context(), notaryServer.URL, "admin", path.Join(endpoint, "library/busybox")) assert.Nil(t, err, fmt.Sprintf("Unexpected error: %v", err)) assert.Equal(t, 1, len(targets), "") assert.Equal(t, "latest-signed", targets[0].Tag, "") - targets, err = GetTargets(notaryServer.URL, "admin", path.Join(endpoint, "library/notexist")) + targets, err = GetTargets(orm.Context(), notaryServer.URL, "admin", path.Join(endpoint, "library/notexist")) assert.Nil(t, err, fmt.Sprintf("Unexpected error: %v", err)) assert.Equal(t, 0, len(targets), "Targets list should be empty for non exist repo.") } diff --git a/src/pkg/task/task.go b/src/pkg/task/task.go index 945f734da..b5a1b2e96 100644 --- a/src/pkg/task/task.go +++ b/src/pkg/task/task.go @@ -18,11 +18,11 @@ import ( "context" "encoding/json" "fmt" + "github.com/goharbor/harbor/src/controller/config" "time" cjob "github.com/goharbor/harbor/src/common/job" "github.com/goharbor/harbor/src/common/job/models" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/q" diff --git a/src/pkg/token/options.go b/src/pkg/token/options.go index e05a33065..4058f913d 100644 --- a/src/pkg/token/options.go +++ b/src/pkg/token/options.go @@ -3,10 +3,10 @@ package token import ( "crypto/rsa" "fmt" + "github.com/goharbor/harbor/src/controller/config" "io/ioutil" "github.com/dgrijalva/jwt-go" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" ) diff --git a/src/pkg/token/token_test.go b/src/pkg/token/token_test.go index 51b316f3f..8e975ab85 100644 --- a/src/pkg/token/token_test.go +++ b/src/pkg/token/token_test.go @@ -1,12 +1,12 @@ package token import ( + "github.com/goharbor/harbor/src/controller/config" "os" "testing" "time" jwt "github.com/dgrijalva/jwt-go" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/pkg/permission/types" robot_claim "github.com/goharbor/harbor/src/pkg/token/claims/robot" "github.com/stretchr/testify/assert" diff --git a/src/server/middleware/blob/head_blob.go b/src/server/middleware/blob/head_blob.go index 9c4a6615c..8079dd8ee 100644 --- a/src/server/middleware/blob/head_blob.go +++ b/src/server/middleware/blob/head_blob.go @@ -3,7 +3,7 @@ package blob import ( "fmt" "github.com/goharbor/harbor/src/controller/blob" - "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/errors" lib_http "github.com/goharbor/harbor/src/lib/http" diff --git a/src/server/middleware/blob/util.go b/src/server/middleware/blob/util.go index b2b26e0e8..b77b4f1da 100644 --- a/src/server/middleware/blob/util.go +++ b/src/server/middleware/blob/util.go @@ -3,7 +3,7 @@ package blob import ( "fmt" "github.com/goharbor/harbor/src/controller/blob" - "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/blob/models" diff --git a/src/server/middleware/csrf/csrf.go b/src/server/middleware/csrf/csrf.go index fcfb505f3..62f71d9a3 100644 --- a/src/server/middleware/csrf/csrf.go +++ b/src/server/middleware/csrf/csrf.go @@ -1,6 +1,7 @@ package csrf import ( + "github.com/goharbor/harbor/src/controller/config" lib_http "github.com/goharbor/harbor/src/lib/http" "net/http" "os" @@ -8,7 +9,6 @@ import ( "sync" "github.com/goharbor/harbor/src/common/utils" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" diff --git a/src/server/middleware/csrf/csrf_test.go b/src/server/middleware/csrf/csrf_test.go index 6225990cb..c87f7065b 100644 --- a/src/server/middleware/csrf/csrf_test.go +++ b/src/server/middleware/csrf/csrf_test.go @@ -2,7 +2,8 @@ package csrf import ( "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/controller/config" "github.com/stretchr/testify/assert" "net/http" "net/http/httptest" @@ -11,6 +12,7 @@ import ( ) func TestMain(m *testing.M) { + test.InitDatabaseFromEnv() conf := map[string]interface{}{} config.InitWithSettings(conf) result := m.Run() diff --git a/src/server/middleware/metric/metric.go b/src/server/middleware/metric/metric.go index 71f730af0..2478c074f 100644 --- a/src/server/middleware/metric/metric.go +++ b/src/server/middleware/metric/metric.go @@ -2,12 +2,12 @@ package metric import ( "context" + "github.com/goharbor/harbor/src/controller/config" "net/http" "strconv" "strings" "time" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/metric" ) diff --git a/src/server/middleware/readonly/readonly.go b/src/server/middleware/readonly/readonly.go index be80d6303..ec05f49bf 100644 --- a/src/server/middleware/readonly/readonly.go +++ b/src/server/middleware/readonly/readonly.go @@ -15,10 +15,11 @@ package readonly import ( + config "github.com/goharbor/harbor/src/controller/config" lib_http "github.com/goharbor/harbor/src/lib/http" + "github.com/goharbor/harbor/src/lib/orm" "net/http" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/server/middleware" ) @@ -33,7 +34,7 @@ var ( // DefaultConfig default readonly config DefaultConfig = Config{ ReadOnly: func(r *http.Request) bool { - return config.ReadOnly() + return config.ReadOnly(orm.Context()) }, } diff --git a/src/server/middleware/security/auth_proxy.go b/src/server/middleware/security/auth_proxy.go index 143be670b..74fa2745e 100644 --- a/src/server/middleware/security/auth_proxy.go +++ b/src/server/middleware/security/auth_proxy.go @@ -15,6 +15,8 @@ package security import ( + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" "net/http" "strings" @@ -24,7 +26,6 @@ import ( "github.com/goharbor/harbor/src/common/security" "github.com/goharbor/harbor/src/common/security/local" "github.com/goharbor/harbor/src/core/auth" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/authproxy" @@ -50,7 +51,7 @@ func (a *authProxy) Generate(req *http.Request) security.Context { log.Errorf("user name %s doesn't meet the auth proxy name pattern", proxyUserName) return nil } - httpAuthProxyConf, err := config.HTTPAuthProxySetting() + httpAuthProxyConf, err := config.HTTPAuthProxySetting(orm.Context()) if err != nil { log.Errorf("failed to get auth proxy settings: %v", err) return nil diff --git a/src/server/middleware/security/auth_proxy_test.go b/src/server/middleware/security/auth_proxy_test.go index ac09f5919..ad92951ae 100644 --- a/src/server/middleware/security/auth_proxy_test.go +++ b/src/server/middleware/security/auth_proxy_test.go @@ -18,11 +18,12 @@ import ( "encoding/json" "fmt" "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/controller/config" _ "github.com/goharbor/harbor/src/core/auth/authproxy" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib" + "github.com/goharbor/harbor/src/lib/config/models" + "github.com/goharbor/harbor/src/lib/orm" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "io/ioutil" @@ -50,7 +51,7 @@ func TestAuthProxy(t *testing.T) { common.AUTHMode: common.HTTPAuth, } config.Upload(c) - v, e := config.HTTPAuthProxySetting() + v, e := config.HTTPAuthProxySetting(orm.Context()) require.Nil(t, e) assert.Equal(t, *v, models.HTTPAuthProxy{ Endpoint: "https://auth.proxy/suffix", diff --git a/src/server/middleware/security/idtoken.go b/src/server/middleware/security/idtoken.go index 24fa1257b..918124fb4 100644 --- a/src/server/middleware/security/idtoken.go +++ b/src/server/middleware/security/idtoken.go @@ -15,6 +15,7 @@ package security import ( + "github.com/goharbor/harbor/src/controller/config" "net/http" "strings" @@ -22,7 +23,6 @@ import ( "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/security" "github.com/goharbor/harbor/src/common/security/local" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/oidc" @@ -57,7 +57,7 @@ func (i *idToken) Generate(req *http.Request) security.Context { log.Warning("user matches token's claims is not onboarded.") return nil } - setting, err := config.OIDCSetting() + setting, err := config.OIDCSetting(ctx) if err != nil { log.Errorf("failed to get OIDC settings: %v", err) return nil diff --git a/src/server/middleware/security/robot2.go b/src/server/middleware/security/robot2.go index b358d8c50..ae4ceaf5b 100644 --- a/src/server/middleware/security/robot2.go +++ b/src/server/middleware/security/robot2.go @@ -5,8 +5,8 @@ import ( "github.com/goharbor/harbor/src/common/security" robotCtx "github.com/goharbor/harbor/src/common/security/robot" "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/controller/config" robot_ctl "github.com/goharbor/harbor/src/controller/robot" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/permission/types" @@ -25,12 +25,12 @@ func (r *robot2) Generate(req *http.Request) security.Context { if !ok { return nil } - if !strings.HasPrefix(name, config.RobotPrefix()) { + if !strings.HasPrefix(name, config.RobotPrefix(req.Context())) { return nil } // The robot name can be used as the unique identifier to locate robot as it contains the project name. robots, err := robot_ctl.Ctl.List(req.Context(), q.New(q.KeyWords{ - "name": strings.TrimPrefix(name, config.RobotPrefix()), + "name": strings.TrimPrefix(name, config.RobotPrefix(req.Context())), }), &robot_ctl.Option{ WithPermission: true, }) diff --git a/src/server/middleware/security/robot2_test.go b/src/server/middleware/security/robot2_test.go index 614ba67dd..a36195068 100644 --- a/src/server/middleware/security/robot2_test.go +++ b/src/server/middleware/security/robot2_test.go @@ -2,7 +2,7 @@ package security import ( "github.com/goharbor/harbor/src/common" - core_cfg "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/controller/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "net/http" @@ -13,7 +13,7 @@ func TestRobot2(t *testing.T) { conf := map[string]interface{}{ common.RobotNamePrefix: "robot@", } - core_cfg.InitWithSettings(conf) + config.InitWithSettings(conf) robot := &robot2{} req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1/api/projects/", nil) diff --git a/src/server/middleware/security/secret.go b/src/server/middleware/security/secret.go index 08946cd5f..a2a7c85e8 100644 --- a/src/server/middleware/security/secret.go +++ b/src/server/middleware/security/secret.go @@ -15,12 +15,12 @@ package security import ( + "github.com/goharbor/harbor/src/controller/config" "net/http" commonsecret "github.com/goharbor/harbor/src/common/secret" "github.com/goharbor/harbor/src/common/security" securitysecret "github.com/goharbor/harbor/src/common/security/secret" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/log" ) diff --git a/src/server/middleware/security/secret_test.go b/src/server/middleware/security/secret_test.go index fcaef7c10..ae3086a11 100644 --- a/src/server/middleware/security/secret_test.go +++ b/src/server/middleware/security/secret_test.go @@ -15,11 +15,12 @@ package security import ( + "net/http" + "testing" + commonsecret "github.com/goharbor/harbor/src/common/secret" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "net/http" - "testing" ) func TestSecret(t *testing.T) { diff --git a/src/server/middleware/security/security.go b/src/server/middleware/security/security.go index 65184256c..60c99814b 100644 --- a/src/server/middleware/security/security.go +++ b/src/server/middleware/security/security.go @@ -15,10 +15,11 @@ package security import ( + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" "net/http" "github.com/goharbor/harbor/src/common/security" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/server/middleware" @@ -48,7 +49,7 @@ type generator interface { func Middleware(skippers ...middleware.Skipper) func(http.Handler) http.Handler { return middleware.New(func(w http.ResponseWriter, r *http.Request, next http.Handler) { log := log.G(r.Context()) - mode, err := config.AuthMode() + mode, err := config.AuthMode(orm.Context()) if err == nil { r = r.WithContext(lib.WithAuthMode(r.Context(), mode)) } else { diff --git a/src/server/middleware/security/v2_token_test.go b/src/server/middleware/security/v2_token_test.go index 79d0183d4..63ee3c74c 100644 --- a/src/server/middleware/security/v2_token_test.go +++ b/src/server/middleware/security/v2_token_test.go @@ -2,10 +2,11 @@ package security import ( "fmt" + "github.com/goharbor/harbor/src/controller/config" + "github.com/goharbor/harbor/src/lib/orm" "net/http" "testing" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/service/token" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -17,17 +18,18 @@ func TestGenerate(t *testing.T) { config.Init() vt := &v2Token{} req1, _ := http.NewRequest(http.MethodHead, "/api/2.0/", nil) + ctx := orm.Context() assert.Nil(t, vt.Generate(req1)) req2, _ := http.NewRequest(http.MethodGet, "/v2/library/ubuntu/manifests/v1.0", nil) req2.Header.Set("Authorization", "Bearer 123") assert.Nil(t, vt.Generate(req2)) - mt, err := token.MakeToken("admin", "none", []*registry_token.ResourceActions{}) + mt, err := token.MakeToken(ctx, "admin", "none", []*registry_token.ResourceActions{}) require.Nil(t, err) req3 := req2.Clone(req2.Context()) req3.Header.Set("Authorization", fmt.Sprintf("Bearer %s", mt.Token)) assert.Nil(t, vt.Generate(req3)) req4 := req3.Clone(req3.Context()) - mt2, err2 := token.MakeToken("admin", token.Registry, []*registry_token.ResourceActions{}) + mt2, err2 := token.MakeToken(ctx, "admin", token.Registry, []*registry_token.ResourceActions{}) require.Nil(t, err2) req4.Header.Set("Authorization", fmt.Sprintf("Bearer %s", mt2.Token)) assert.NotNil(t, vt.Generate(req4)) diff --git a/src/server/middleware/session/session.go b/src/server/middleware/session/session.go index 9b6afb879..4c9e9046b 100644 --- a/src/server/middleware/session/session.go +++ b/src/server/middleware/session/session.go @@ -15,7 +15,7 @@ package session import ( - "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/lib" "net/http" ) diff --git a/src/server/middleware/session/session_test.go b/src/server/middleware/session/session_test.go index c747c8e2d..683083ca7 100644 --- a/src/server/middleware/session/session_test.go +++ b/src/server/middleware/session/session_test.go @@ -17,7 +17,7 @@ package session import ( "github.com/astaxie/beego" beegosession "github.com/astaxie/beego/session" - "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/lib" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/src/server/middleware/v2auth/auth.go b/src/server/middleware/v2auth/auth.go index ff2ed6b74..7edf12878 100644 --- a/src/server/middleware/v2auth/auth.go +++ b/src/server/middleware/v2auth/auth.go @@ -19,6 +19,7 @@ import ( "fmt" rbac_project "github.com/goharbor/harbor/src/common/rbac/project" "github.com/goharbor/harbor/src/common/rbac/system" + "github.com/goharbor/harbor/src/controller/config" "net/http" "net/url" "strings" @@ -27,7 +28,6 @@ import ( "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/security" "github.com/goharbor/harbor/src/controller/project" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/service/token" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/errors" diff --git a/src/server/middleware/v2auth/auth_test.go b/src/server/middleware/v2auth/auth_test.go index c2dfc184e..564a70b37 100644 --- a/src/server/middleware/v2auth/auth_test.go +++ b/src/server/middleware/v2auth/auth_test.go @@ -17,6 +17,8 @@ package v2auth import ( "context" "fmt" + testutils "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/controller/config" "net/http" "net/http/httptest" "os" @@ -29,7 +31,6 @@ import ( "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/security" "github.com/goharbor/harbor/src/controller/project" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/pkg/permission/types" securitytesting "github.com/goharbor/harbor/src/testing/common/security" @@ -39,6 +40,7 @@ import ( ) func TestMain(m *testing.M) { + testutils.InitDatabaseFromEnv() ctl := &projecttesting.Controller{} mockGet := func(ctx context.Context, diff --git a/src/server/registry/proxy.go b/src/server/registry/proxy.go index 5fe9448db..67fcd8a33 100644 --- a/src/server/registry/proxy.go +++ b/src/server/registry/proxy.go @@ -16,12 +16,12 @@ package registry import ( "fmt" + "github.com/goharbor/harbor/src/controller/config" "net/http" "net/http/httputil" "net/url" commonhttp "github.com/goharbor/harbor/src/common/http" - "github.com/goharbor/harbor/src/core/config" ) var proxy = newProxy() diff --git a/src/server/route.go b/src/server/route.go index 10a853808..4f00364c8 100644 --- a/src/server/route.go +++ b/src/server/route.go @@ -15,12 +15,12 @@ package server import ( + "github.com/goharbor/harbor/src/controller/config" "net/http" "github.com/astaxie/beego" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/core/api" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/controllers" "github.com/goharbor/harbor/src/core/service/notifications/jobs" "github.com/goharbor/harbor/src/core/service/token" @@ -45,7 +45,6 @@ func registerRoutes() { beego.Router(common.OIDCCallbackPath, &controllers.OIDCController{}, "get:Callback") beego.Router(common.AuthProxyRediretPath, &controllers.AuthProxyController{}, "get:HandleRedirect") - beego.Router("/api/internal/configurations", &api.ConfigAPI{}, "get:GetInternalConfig;put:Put") beego.Router("/api/internal/renameadmin", &api.InternalAPI{}, "post:RenameAdmin") beego.Router("/api/internal/switchquota", &api.InternalAPI{}, "put:SwitchQuota") beego.Router("/api/internal/syncquota", &api.InternalAPI{}, "post:SyncQuota") diff --git a/src/server/v2.0/handler/base.go b/src/server/v2.0/handler/base.go index 3aa7d78f0..580e9aac4 100644 --- a/src/server/v2.0/handler/base.go +++ b/src/server/v2.0/handler/base.go @@ -144,6 +144,15 @@ func (b *BaseAPI) RequireAuthenticated(ctx context.Context) error { return nil } +// RequireSolutionUserAccess check if current user is internal service +func (b *BaseAPI) RequireSolutionUserAccess(ctx context.Context) error { + sec, exist := security.FromContext(ctx) + if !exist || !sec.IsSolutionUser() { + return errors.UnauthorizedError(nil).WithMessage("only internal service is allowed to call this API") + } + return nil +} + // BuildQuery builds the query model according to the query string func (b *BaseAPI) BuildQuery(ctx context.Context, query, sort *string, pageNumber, pageSize *int64) (*q.Query, error) { var ( diff --git a/src/server/v2.0/handler/config.go b/src/server/v2.0/handler/config.go new file mode 100644 index 000000000..7fc6dd416 --- /dev/null +++ b/src/server/v2.0/handler/config.go @@ -0,0 +1,112 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handler + +import ( + "context" + "encoding/json" + "github.com/go-openapi/runtime/middleware" + "github.com/goharbor/harbor/src/common/rbac" + "github.com/goharbor/harbor/src/common/security" + "github.com/goharbor/harbor/src/controller/config" + libCfg "github.com/goharbor/harbor/src/lib/config" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/server/v2.0/models" + "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/configure" +) + +type configAPI struct { + BaseAPI + controller config.Controller +} + +func newConfigAPI() *configAPI { + return &configAPI{controller: config.Ctl} +} + +func (c *configAPI) GetConfigurations(ctx context.Context, params configure.GetConfigurationsParams) middleware.Responder { + if sec, exist := security.FromContext(ctx); exist { + if sec.IsSolutionUser() { + cfg, err := c.controller.AllConfigs(ctx) + if err != nil { + return c.SendError(ctx, err) + } + var result []byte + result, err = json.Marshal(cfg) + if err != nil { + return c.SendError(ctx, err) + } + var cfgResp models.ConfigurationsResponse + err = json.Unmarshal(result, &cfgResp) + if err != nil { + return c.SendError(ctx, err) + } + return configure.NewGetConfigurationsOK().WithPayload(&cfgResp) + } + } + if err := c.RequireSystemAccess(ctx, rbac.ActionRead, rbac.ResourceConfiguration); err != nil { + return c.SendError(ctx, err) + } + cfg, err := c.controller.UserConfigs(ctx) + if err != nil { + return c.SendError(ctx, err) + } + payload, err := toResponseModel(cfg) + if err != nil { + c.SendError(ctx, err) + } + return configure.NewGetConfigurationsOK().WithPayload(payload) +} + +func (c *configAPI) PutConfigurations(ctx context.Context, params configure.PutConfigurationsParams) middleware.Responder { + if err := c.RequireSystemAccess(ctx, rbac.ActionUpdate, rbac.ResourceConfiguration); err != nil { + return c.SendError(ctx, err) + } + if params.Configurations == nil { + return c.SendError(ctx, errors.BadRequestError(nil).WithMessage("Missing configure item")) + } + conf := params.Configurations + err := c.controller.UpdateUserConfigs(ctx, conf) + if err != nil { + return c.SendError(ctx, err) + } + return configure.NewPutConfigurationsOK() +} + +func (c *configAPI) GetInternalconfig(ctx context.Context, params configure.GetInternalconfigParams) middleware.Responder { + if err := c.RequireSolutionUserAccess(ctx); err != nil { + return c.SendError(ctx, err) + } + cfg, err := c.controller.AllConfigs(ctx) + resultCfg, err := config.ConvertForGet(ctx, cfg, true) + if err != nil { + return c.SendError(ctx, err) + } + return configure.NewGetInternalconfigOK().WithPayload(resultCfg) +} + +func toResponseModel(cfg map[string]*libCfg.Value) (*models.ConfigurationsResponse, error) { + var result []byte + result, err := json.Marshal(cfg) + if err != nil { + return nil, err + } + var cfgResp models.ConfigurationsResponse + err = json.Unmarshal(result, &cfgResp) + if err != nil { + return nil, err + } + return &cfgResp, nil +} diff --git a/src/server/v2.0/handler/gc.go b/src/server/v2.0/handler/gc.go index ec1ce9062..5f74237bf 100644 --- a/src/server/v2.0/handler/gc.go +++ b/src/server/v2.0/handler/gc.go @@ -4,13 +4,13 @@ import ( "context" "encoding/json" "fmt" + "github.com/goharbor/harbor/src/controller/config" "os" "strings" "github.com/go-openapi/runtime/middleware" "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/controller/gc" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/task" diff --git a/src/server/v2.0/handler/handler.go b/src/server/v2.0/handler/handler.go index d4b23b568..30dbcb28a 100644 --- a/src/server/v2.0/handler/handler.go +++ b/src/server/v2.0/handler/handler.go @@ -56,6 +56,7 @@ func New() http.Handler { ImmutableAPI: newImmutableAPI(), OidcAPI: newOIDCAPI(), SystemCVEAllowlistAPI: newSystemCVEAllowListAPI(), + ConfigureAPI: newConfigAPI(), }) if err != nil { log.Fatal(err) diff --git a/src/server/v2.0/handler/project.go b/src/server/v2.0/handler/project.go index eebc6c88d..d19f6358e 100644 --- a/src/server/v2.0/handler/project.go +++ b/src/server/v2.0/handler/project.go @@ -17,6 +17,7 @@ package handler import ( "context" "fmt" + "github.com/goharbor/harbor/src/controller/config" "strconv" "strings" "sync" @@ -36,7 +37,6 @@ import ( "github.com/goharbor/harbor/src/controller/retention" "github.com/goharbor/harbor/src/controller/scanner" "github.com/goharbor/harbor/src/core/api" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" @@ -90,7 +90,7 @@ func (a *projectAPI) CreateProject(ctx context.Context, params operation.CreateP return a.SendError(ctx, err) } - onlyAdmin, err := config.OnlyAdminCreateProject() + onlyAdmin, err := config.OnlyAdminCreateProject(ctx) if err != nil { return a.SendError(ctx, fmt.Errorf("failed to determine whether only admin can create projects: %v", err)) } @@ -108,10 +108,10 @@ func (a *projectAPI) CreateProject(ctx context.Context, params operation.CreateP } // populate storage limit - if config.QuotaPerProjectEnable() { + if config.QuotaPerProjectEnable(ctx) { // the security context is not sys admin, set the StorageLimit the global StoragePerProject if req.StorageLimit == nil || *req.StorageLimit == 0 || !a.isSysAdmin(ctx, rbac.ActionCreate) { - setting, err := config.QuotaSetting() + setting, err := config.QuotaSetting(ctx) if err != nil { log.Errorf("failed to get quota setting: %v", err) return a.SendError(ctx, fmt.Errorf("failed to get quota setting: %v", err)) @@ -676,7 +676,7 @@ func (a *projectAPI) isSysAdmin(ctx context.Context, action rbac.Action) bool { } func getProjectQuotaSummary(ctx context.Context, p *project.Project, summary *models.ProjectSummary) { - if !config.QuotaPerProjectEnable() { + if !config.QuotaPerProjectEnable(ctx) { log.Debug("Quota per project disabled") return } diff --git a/src/server/v2.0/handler/robot.go b/src/server/v2.0/handler/robot.go index b4d2391cf..5487b71e7 100644 --- a/src/server/v2.0/handler/robot.go +++ b/src/server/v2.0/handler/robot.go @@ -7,8 +7,8 @@ import ( "github.com/go-openapi/strfmt" "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/controller/robot" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/errors" pkg "github.com/goharbor/harbor/src/pkg/robot/model" @@ -312,8 +312,8 @@ func (rAPI *robotAPI) updateV2Robot(ctx context.Context, params operation.Update if params.Robot.Duration == -1 { r.ExpiresAt = -1 } else if params.Robot.Duration == 0 { - r.Duration = int64(config.RobotTokenDuration()) - r.ExpiresAt = r.CreationTime.AddDate(0, 0, config.RobotTokenDuration()).Unix() + r.Duration = int64(config.RobotTokenDuration(ctx)) + r.ExpiresAt = r.CreationTime.AddDate(0, 0, config.RobotTokenDuration(ctx)).Unix() } else { r.ExpiresAt = r.CreationTime.AddDate(0, 0, int(params.Robot.Duration)).Unix() } diff --git a/src/server/v2.0/handler/search.go b/src/server/v2.0/handler/search.go index 9b12d72d2..b736b101a 100644 --- a/src/server/v2.0/handler/search.go +++ b/src/server/v2.0/handler/search.go @@ -26,10 +26,10 @@ import ( "github.com/goharbor/harbor/src/common/security/local" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/controller/artifact" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/controller/project" "github.com/goharbor/harbor/src/controller/repository" "github.com/goharbor/harbor/src/core/api" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" diff --git a/src/server/v2.0/route/legacy.go b/src/server/v2.0/route/legacy.go index 2e40d9004..1ffb6ac98 100755 --- a/src/server/v2.0/route/legacy.go +++ b/src/server/v2.0/route/legacy.go @@ -16,8 +16,8 @@ package route import ( "github.com/astaxie/beego" + "github.com/goharbor/harbor/src/controller/config" "github.com/goharbor/harbor/src/core/api" - "github.com/goharbor/harbor/src/core/config" ) // RegisterRoutes for Harbor legacy APIs @@ -36,8 +36,6 @@ func registerLegacyRoutes() { beego.Router("/api/"+version+"/health", &api.HealthAPI{}, "get:CheckHealth") beego.Router("/api/"+version+"/projects/:id([0-9]+)/metadatas/?:name", &api.MetadataAPI{}, "get:Get") beego.Router("/api/"+version+"/projects/:id([0-9]+)/metadatas/", &api.MetadataAPI{}, "post:Post") - - beego.Router("/api/"+version+"/configurations", &api.ConfigAPI{}, "get:Get;put:Put") beego.Router("/api/"+version+"/statistics", &api.StatisticAPI{}) beego.Router("/api/"+version+"/labels", &api.LabelAPI{}, "post:Post;get:List") beego.Router("/api/"+version+"/labels/:id([0-9]+)", &api.LabelAPI{}, "get:Get;put:Put;delete:Delete") diff --git a/src/testing/controller/controller.go b/src/testing/controller/controller.go index dd35e7ab1..2a45b4715 100644 --- a/src/testing/controller/controller.go +++ b/src/testing/controller/controller.go @@ -25,3 +25,4 @@ package controller //go:generate mockery --case snake --dir ../../controller/robot --name Controller --output ./robot --outpkg robot //go:generate mockery --case snake --dir ../../controller/proxy --name RemoteInterface --output ./proxy --outpkg proxy //go:generate mockery --case snake --dir ../../controller/retention --name Controller --output ./retention --outpkg retention +//go:generate mockery --case snake --dir ../../controller/config --name Controller --output ./config --outpkg config diff --git a/src/testing/suite.go b/src/testing/suite.go index f6bd2bb13..6e02394d6 100644 --- a/src/testing/suite.go +++ b/src/testing/suite.go @@ -16,6 +16,7 @@ package testing import ( "context" + "github.com/goharbor/harbor/src/controller/config" "io" "math/rand" "net/http" @@ -26,7 +27,6 @@ import ( o "github.com/astaxie/beego/orm" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/orm" "github.com/opencontainers/go-digest" diff --git a/tests/apitests/python/library/base.py b/tests/apitests/python/library/base.py index b08556543..492023855 100644 --- a/tests/apitests/python/library/base.py +++ b/tests/apitests/python/library/base.py @@ -1,194 +1,196 @@ -# -*- coding: utf-8 -*- -import os -import sys -import time -import subprocess -import client -import swagger_client -import v2_swagger_client -try: - from urllib import getproxies -except ImportError: - from urllib.request import getproxies - -class Server: - def __init__(self, endpoint, verify_ssl): - self.endpoint = endpoint - self.verify_ssl = verify_ssl - -class Credential: - def __init__(self, type, username, password): - self.type = type - self.username = username - self.password = password - -def get_endpoint(): - harbor_server = os.environ.get("HARBOR_HOST", "localhost:8080") - return os.environ.get("HARBOR_HOST_SCHEMA", "https")+ "://"+harbor_server+"/api/v2.0" - -def _create_client(server, credential, debug, api_type="products"): - cfg = None - if api_type in ('projectv2', 'artifact', 'repository', 'scanner', 'scan', 'scanall', 'preheat', 'quota', 'replication', 'registry', 'robot', 'gc', 'retention', "immutable", "system_cve_allowlist"): - cfg = v2_swagger_client.Configuration() - else: - cfg = swagger_client.Configuration() - - cfg.host = server.endpoint - cfg.verify_ssl = server.verify_ssl - # support basic auth only for now - cfg.username = credential.username - cfg.password = credential.password - cfg.debug = debug - - proxies = getproxies() - proxy = proxies.get('http', proxies.get('all', None)) - if proxy: - cfg.proxy = proxy - - if cfg.username is None and cfg.password is None: - # returns {} for auth_settings for anonymous access - import types - cfg.auth_settings = types.MethodType(lambda self: {}, cfg) - - return { - "chart": client.ChartRepositoryApi(client.ApiClient(cfg)), - "products": swagger_client.ProductsApi(swagger_client.ApiClient(cfg)), - "projectv2": v2_swagger_client.ProjectApi(v2_swagger_client.ApiClient(cfg)), - "artifact": v2_swagger_client.ArtifactApi(v2_swagger_client.ApiClient(cfg)), - "preheat": v2_swagger_client.PreheatApi(v2_swagger_client.ApiClient(cfg)), - "quota": v2_swagger_client.QuotaApi(v2_swagger_client.ApiClient(cfg)), - "repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)), - "scan": v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)), - "scanall": v2_swagger_client.ScanAllApi(v2_swagger_client.ApiClient(cfg)), - "scanner": v2_swagger_client.ScannerApi(v2_swagger_client.ApiClient(cfg)), - "replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)), - "registry": v2_swagger_client.RegistryApi(v2_swagger_client.ApiClient(cfg)), - "robot": v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)), - "gc": v2_swagger_client.GcApi(v2_swagger_client.ApiClient(cfg)), - "retention": v2_swagger_client.RetentionApi(v2_swagger_client.ApiClient(cfg)), - "immutable": v2_swagger_client.ImmutableApi(v2_swagger_client.ApiClient(cfg)), - "system_cve_allowlist": v2_swagger_client.SystemCVEAllowlistApi(v2_swagger_client.ApiClient(cfg)), - }.get(api_type,'Error: Wrong API type') - -def _assert_status_code(expect_code, return_code, err_msg = r"HTTPS status code s not as we expected. Expected {}, while actual HTTPS status code is {}."): - if str(return_code) != str(expect_code): - raise Exception(err_msg.format(expect_code, return_code)) - -def _assert_status_body(expect_status_body, returned_status_body): - if str(returned_status_body.strip()).lower().find(expect_status_body.lower()) < 0: - raise Exception(r"HTTPS status body s not as we expected. Expected {}, while actual HTTPS status body is {}.".format(expect_status_body, returned_status_body)) - -def _random_name(prefix): - return "%s-%d" % (prefix, int(round(time.time() * 1000))) - -def _get_id_from_header(header): - try: - location = header["Location"] - return int(location.split("/")[-1]) - except Exception: - return None - -def _get_string_from_unicode(udata): - result='' - for u in udata: - tmp = u.encode('utf8') - result = result + tmp.strip('\n\r\t') - return result - -def restart_process(process): - if process == "dockerd": - full_process_name = process - elif process == "containerd": - full_process_name = "/usr/local/bin/containerd" - else: - raise Exception("Please input dockerd or containerd for process retarting.") - run_command_with_popen("ps aux |grep " + full_process_name) - for i in range(10): - pid = run_command_with_popen(["pidof " + full_process_name]) - if pid in [None, ""]: - break - run_command_with_popen(["kill " + str(pid)]) - time.sleep(3) - - run_command_with_popen("ps aux |grep " + full_process_name) - run_command_with_popen("rm -rf /var/lib/" + process + "/*") - run_command_with_popen(full_process_name + " > ./daemon-local.log 2>&1 &") - time.sleep(3) - pid = run_command_with_popen(["pidof " + full_process_name]) - if pid in [None, ""]: - raise Exception("Failed to start process {}.".format(full_process_name)) - run_command_with_popen("ps aux |grep " + full_process_name) - - - -def run_command_with_popen(command): - print("Command: ", command) - - try: - proc = subprocess.Popen(command, universal_newlines=True, shell=True, - stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - output, errors = proc.communicate() - except Exception as e: - print("Run command caught exception:", e) - output = None - else: - print(proc.returncode, errors, output) - finally: - proc.stdout.close() - print("output: ", output) - return output - -def run_command(command, expected_error_message = None): - print("Command: ", subprocess.list2cmdline(command)) - try: - output = subprocess.check_output(command, - stderr=subprocess.STDOUT, - universal_newlines=True) - except subprocess.CalledProcessError as e: - print("Run command error:", str(e)) - print("expected_error_message:", expected_error_message) - if expected_error_message is not None: - if str(e.output).lower().find(expected_error_message.lower()) < 0: - raise Exception(r"Error message {} is not as expected {}".format(str(e.output), expected_error_message)) - else: - raise Exception('Error: Exited with error code: %s. Output:%s'% (e.returncode, e.output)) - else: - print("output:", output) - return output - -class Base(object): - def __init__(self, server=None, credential=None, debug=True, api_type="products"): - if server is None: - server = Server(endpoint=get_endpoint(), verify_ssl=False) - if not isinstance(server.verify_ssl, bool): - server.verify_ssl = server.verify_ssl == "True" - - if credential is None: - credential = Credential(type="basic_auth", username="admin", password="Harbor12345") - - self.server = server - self.credential = credential - self.debug = debug - self.api_type = api_type - self.client = _create_client(server, credential, debug, api_type=api_type) - - def _get_client(self, **kwargs): - if len(kwargs) == 0: - return self.client - - server = self.server - if "endpoint" in kwargs: - server.endpoint = kwargs.get("endpoint") - if "verify_ssl" in kwargs: - if not isinstance(kwargs.get("verify_ssl"), bool): - server.verify_ssl = kwargs.get("verify_ssl") == "True" - else: - server.verify_ssl = kwargs.get("verify_ssl") - - credential = Credential( - kwargs.get("type", self.credential.type), - kwargs.get("username", self.credential.username), - kwargs.get("password", self.credential.password), - ) - - return _create_client(server, credential, self.debug, kwargs.get('api_type', self.api_type)) +# -*- coding: utf-8 -*- +import os +import subprocess +import time + +import client +import swagger_client +import v2_swagger_client + +try: + from urllib import getproxies +except ImportError: + from urllib.request import getproxies + +class Server: + def __init__(self, endpoint, verify_ssl): + self.endpoint = endpoint + self.verify_ssl = verify_ssl + +class Credential: + def __init__(self, type, username, password): + self.type = type + self.username = username + self.password = password + +def get_endpoint(): + harbor_server = os.environ.get("HARBOR_HOST", "localhost:8080") + return os.environ.get("HARBOR_HOST_SCHEMA", "https")+ "://"+harbor_server+"/api/v2.0" + +def _create_client(server, credential, debug, api_type="products"): + cfg = None + if api_type in ('projectv2', 'artifact', 'repository', 'scanner', 'scan', 'scanall', 'preheat', 'quota', 'replication', 'registry', 'robot', 'gc', 'retention', "immutable", "system_cve_allowlist", "configure"): + cfg = v2_swagger_client.Configuration() + else: + cfg = swagger_client.Configuration() + + cfg.host = server.endpoint + cfg.verify_ssl = server.verify_ssl + # support basic auth only for now + cfg.username = credential.username + cfg.password = credential.password + cfg.debug = debug + + proxies = getproxies() + proxy = proxies.get('http', proxies.get('all', None)) + if proxy: + cfg.proxy = proxy + + if cfg.username is None and cfg.password is None: + # returns {} for auth_settings for anonymous access + import types + cfg.auth_settings = types.MethodType(lambda self: {}, cfg) + + return { + "chart": client.ChartRepositoryApi(client.ApiClient(cfg)), + "products": swagger_client.ProductsApi(swagger_client.ApiClient(cfg)), + "projectv2": v2_swagger_client.ProjectApi(v2_swagger_client.ApiClient(cfg)), + "artifact": v2_swagger_client.ArtifactApi(v2_swagger_client.ApiClient(cfg)), + "preheat": v2_swagger_client.PreheatApi(v2_swagger_client.ApiClient(cfg)), + "quota": v2_swagger_client.QuotaApi(v2_swagger_client.ApiClient(cfg)), + "repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)), + "scan": v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)), + "scanall": v2_swagger_client.ScanAllApi(v2_swagger_client.ApiClient(cfg)), + "scanner": v2_swagger_client.ScannerApi(v2_swagger_client.ApiClient(cfg)), + "replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)), + "registry": v2_swagger_client.RegistryApi(v2_swagger_client.ApiClient(cfg)), + "robot": v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)), + "gc": v2_swagger_client.GcApi(v2_swagger_client.ApiClient(cfg)), + "retention": v2_swagger_client.RetentionApi(v2_swagger_client.ApiClient(cfg)), + "immutable": v2_swagger_client.ImmutableApi(v2_swagger_client.ApiClient(cfg)), + "system_cve_allowlist": v2_swagger_client.SystemCVEAllowlistApi(v2_swagger_client.ApiClient(cfg)), + "configure": v2_swagger_client.ConfigureApi(v2_swagger_client.ApiClient(cfg)), + }.get(api_type,'Error: Wrong API type') + +def _assert_status_code(expect_code, return_code, err_msg = r"HTTPS status code s not as we expected. Expected {}, while actual HTTPS status code is {}."): + if str(return_code) != str(expect_code): + raise Exception(err_msg.format(expect_code, return_code)) + +def _assert_status_body(expect_status_body, returned_status_body): + if str(returned_status_body.strip()).lower().find(expect_status_body.lower()) < 0: + raise Exception(r"HTTPS status body s not as we expected. Expected {}, while actual HTTPS status body is {}.".format(expect_status_body, returned_status_body)) + +def _random_name(prefix): + return "%s-%d" % (prefix, int(round(time.time() * 1000))) + +def _get_id_from_header(header): + try: + location = header["Location"] + return int(location.split("/")[-1]) + except Exception: + return None + +def _get_string_from_unicode(udata): + result='' + for u in udata: + tmp = u.encode('utf8') + result = result + tmp.strip('\n\r\t') + return result + +def restart_process(process): + if process == "dockerd": + full_process_name = process + elif process == "containerd": + full_process_name = "/usr/local/bin/containerd" + else: + raise Exception("Please input dockerd or containerd for process retarting.") + run_command_with_popen("ps aux |grep " + full_process_name) + for i in range(10): + pid = run_command_with_popen(["pidof " + full_process_name]) + if pid in [None, ""]: + break + run_command_with_popen(["kill " + str(pid)]) + time.sleep(3) + + run_command_with_popen("ps aux |grep " + full_process_name) + run_command_with_popen("rm -rf /var/lib/" + process + "/*") + run_command_with_popen(full_process_name + " > ./daemon-local.log 2>&1 &") + time.sleep(3) + pid = run_command_with_popen(["pidof " + full_process_name]) + if pid in [None, ""]: + raise Exception("Failed to start process {}.".format(full_process_name)) + run_command_with_popen("ps aux |grep " + full_process_name) + + + +def run_command_with_popen(command): + print("Command: ", command) + + try: + proc = subprocess.Popen(command, universal_newlines=True, shell=True, + stdout=subprocess.PIPE,stderr=subprocess.STDOUT) + output, errors = proc.communicate() + except Exception as e: + print("Run command caught exception:", e) + output = None + else: + print(proc.returncode, errors, output) + finally: + proc.stdout.close() + print("output: ", output) + return output + +def run_command(command, expected_error_message = None): + print("Command: ", subprocess.list2cmdline(command)) + try: + output = subprocess.check_output(command, + stderr=subprocess.STDOUT, + universal_newlines=True) + except subprocess.CalledProcessError as e: + print("Run command error:", str(e)) + print("expected_error_message:", expected_error_message) + if expected_error_message is not None: + if str(e.output).lower().find(expected_error_message.lower()) < 0: + raise Exception(r"Error message {} is not as expected {}".format(str(e.output), expected_error_message)) + else: + raise Exception('Error: Exited with error code: %s. Output:%s'% (e.returncode, e.output)) + else: + print("output:", output) + return output + +class Base(object): + def __init__(self, server=None, credential=None, debug=True, api_type="products"): + if server is None: + server = Server(endpoint=get_endpoint(), verify_ssl=False) + if not isinstance(server.verify_ssl, bool): + server.verify_ssl = server.verify_ssl == "True" + + if credential is None: + credential = Credential(type="basic_auth", username="admin", password="Harbor12345") + + self.server = server + self.credential = credential + self.debug = debug + self.api_type = api_type + self.client = _create_client(server, credential, debug, api_type=api_type) + + def _get_client(self, **kwargs): + if len(kwargs) == 0: + return self.client + + server = self.server + if "endpoint" in kwargs: + server.endpoint = kwargs.get("endpoint") + if "verify_ssl" in kwargs: + if not isinstance(kwargs.get("verify_ssl"), bool): + server.verify_ssl = kwargs.get("verify_ssl") == "True" + else: + server.verify_ssl = kwargs.get("verify_ssl") + + credential = Credential( + kwargs.get("type", self.credential.type), + kwargs.get("username", self.credential.username), + kwargs.get("password", self.credential.password), + ) + + return _create_client(server, credential, self.debug, kwargs.get('api_type', self.api_type)) diff --git a/tests/apitests/python/library/configurations.py b/tests/apitests/python/library/configurations.py index 715b91ae4..caec7d65c 100644 --- a/tests/apitests/python/library/configurations.py +++ b/tests/apitests/python/library/configurations.py @@ -1,28 +1,30 @@ # -*- coding: utf-8 -*- -import base -import swagger_client from swagger_client.rest import ApiException +from v2_swagger_client.rest import ApiException + +import base + def set_configurations(client, expect_status_code = 200, expect_response_body = None, **config): - conf = swagger_client.Configurations() + conf = {} - if "project_creation_restriction" in config: - conf.project_creation_restriction = config.get("project_creation_restriction") - if "token_expiration" in config: - conf.token_expiration = config.get("token_expiration") - if "ldap_filter" in config: - conf.ldap_filter = config.get("ldap_filter") - if "ldap_group_attribute_name" in config: - conf.ldap_group_attribute_name = config.get("ldap_group_attribute_name") + if "project_creation_restriction" in config and config.get("project_creation_restriction") is not None: + conf["project_creation_restriction"] = config.get("project_creation_restriction") + if "token_expiration" in config and config.get("token_expiration") is not None: + conf["token_expiration"] = config.get("token_expiration") + if "ldap_filter" in config and config.get("ldap_filter") is not None: + conf["ldap_filter"] = config.get("ldap_filter") + if "ldap_group_attribute_name" in config and config.get("ldap_group_attribute_name") is not None: + conf["ldap_group_attribute_name"] = config.get("ldap_group_attribute_name") if "ldap_group_base_dn" in config: - conf.ldap_group_base_dn = config.get("ldap_group_base_dn") - if "ldap_group_search_filter" in config: - conf.ldap_group_search_filter = config.get("ldap_group_search_filter") - if "ldap_group_search_scope" in config: - conf.ldap_group_search_scope = config.get("ldap_group_search_scope") - if "ldap_group_admin_dn" in config: - conf.ldap_group_admin_dn = config.get("ldap_group_admin_dn") + conf["ldap_group_base_dn"] = config.get("ldap_group_base_dn") + if "ldap_group_search_filter" in config and config.get("ldap_group_search_filter") is not None: + conf["ldap_group_search_filter"] = config.get("ldap_group_search_filter") + if "ldap_group_search_scope" in config and config.get("ldap_group_search_scope") is not None: + conf["ldap_group_search_scope"] = config.get("ldap_group_search_scope") + if "ldap_group_admin_dn" in config and config.get("ldap_group_admin_dn") is not None: + conf["ldap_group_admin_dn"] = config.get("ldap_group_admin_dn") try: _, status_code, _ = client.configurations_put_with_http_info(conf) @@ -34,7 +36,10 @@ def set_configurations(client, expect_status_code = 200, expect_response_body = base._assert_status_code(expect_status_code, status_code) -class Configurations(base.Base): +class Configurations(base.Base, object): + def __init__(self): + super(Configurations,self).__init__(api_type = "configure") + def get_configurations(self, item_name = None, expect_status_code = 200, expect_response_body = None, **kwargs): client = self._get_client(**kwargs) diff --git a/tests/apitests/python/test_user_group.py b/tests/apitests/python/test_user_group.py index f727cd0ad..53f8c4427 100644 --- a/tests/apitests/python/test_user_group.py +++ b/tests/apitests/python/test_user_group.py @@ -12,18 +12,19 @@ from __future__ import absolute_import + import os import sys + sys.path.append(os.environ["SWAGGER_CLIENT_PATH"]) import unittest import testutils +from testutils import ADMIN_CLIENT -import swagger_client -from swagger_client.rest import ApiException from swagger_client.models.user_group import UserGroup -from swagger_client.models.configurations import Configurations +from library.configurations import Configurations from pprint import pprint #Testcase @@ -36,8 +37,9 @@ class TestUserGroup(unittest.TestCase): product_api = testutils.GetProductApi("admin", "Harbor12345") groupId = 0 def setUp(self): - result = self.product_api.configurations_put(configurations=Configurations(ldap_group_attribute_name="cn", ldap_group_base_dn="ou=groups,dc=example,dc=com", ldap_group_search_filter="objectclass=groupOfNames", ldap_group_search_scope=2)) - pprint(result) + self.conf= Configurations() + self.conf.set_configurations_of_ldap(ldap_filter="", ldap_group_attribute_name="cn", ldap_group_base_dn="ou=groups,dc=example,dc=com", + ldap_group_search_filter="objectclass=groupOfNames", ldap_group_search_scope=2, **ADMIN_CLIENT) pass def tearDown(self): diff --git a/tests/apitests/python/testutils.py b/tests/apitests/python/testutils.py index 8368e5728..fa828927b 100644 --- a/tests/apitests/python/testutils.py +++ b/tests/apitests/python/testutils.py @@ -1,4 +1,3 @@ -import time import os import sys import warnings @@ -15,9 +14,7 @@ path=os.getcwd() + "/tests/apitests/python/" sys.path.insert(0, path) print(sys.path) import v2_swagger_client -from swagger_client.rest import ApiException import swagger_client.models -from pprint import pprint admin_user = "admin" admin_pwd = "Harbor12345" @@ -60,6 +57,19 @@ def GetRepositoryApi(username, password, harbor_server= os.environ.get("HARBOR_H api_instance = v2_swagger_client.RepositoryApi(api_client) return api_instance +def GetConfigureApi(username, password, harbor_server= os.environ.get("HARBOR_HOST", '')): + + cfg = v2_swagger_client.Configuration() + cfg.host = "https://"+harbor_server+"/api/v2.0" + cfg.username = username + cfg.password = password + cfg.verify_ssl = False + cfg.debug = True + api_client = v2_swagger_client.ApiClient(cfg) + api_instance = v2_swagger_client.Configuration(api_client) + return api_instance + + class TestResult(object): def __init__(self): self.num_errors = 0